1060 files changed, 183837 insertions, 48879 deletions

diff --git a/.gitignore b/.gitignore
index c5371b264..ff253c1da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,26 +8,41 @@ install-sh
 ltmain.sh
 Makefile.in
 missing
+py-compile
 *.sw?
 *~
-*lo
-*la
-*o
+*.lo
+*.la
+*.o
+*.tar.gz
+*.rpm
 .libs
+.deps
 Makefile
 stamp-h1
 
 # Generated files
-extras/init.d/glusterfs-server.plist
-extras/init.d/glusterfsd-Debian
-extras/init.d/glusterfsd-Redhat
-extras/init.d/glusterfsd-SuSE
-glusterfs.spec
-libtool
-xlators/mount/fuse/utils/mount.glusterfs
-xlators/mount/fuse/utils/mount_glusterfs
+api/examples/__init__.py*
+api/examples/setup.py
 argp-standalone/libargp.a
+contrib/uuid/uuid_types.h
+extras/init.d/glusterd-Debian
+extras/init.d/glusterd-Redhat
+extras/init.d/glusterd-SuSE
+extras/init.d/glusterd.plist
+extras/ocf/glusterd
+extras/ocf/volume
+extras/who-wrote-glusterfs/gitdm
+glusterfs-api.pc
+glusterfs.spec
 glusterfsd/src/glusterfsd
+libgfchangelog.pc
 libglusterfs/src/spec.lex.c
 libglusterfs/src/y.tab.c
 libglusterfs/src/y.tab.h
+libtool
+run-tests.sh
+ufo/.tox
+ufo/test/unit/.coverage
+xlators/mount/fuse/utils/mount.glusterfs
+xlators/mount/fuse/utils/mount_glusterfs
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 000000000..6bcd95dea
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,31 @@
+# .mailmap, see 'git short-log --help' for details
+#
+# Listing of contributors that filed patches with different email addresses.
+# Format: <name> <main-email> <alias> [<alias> ...]
+#
+
+Amar Tumballi <amarts@redhat.com> <amar@gluster.com> <amar@del.gluster.com>
+Anand Avati <avati@redhat.com> <avati@gluster.com> <avati@dev.gluster.com> <avati@amp.gluster.com> <avati@blackhole.gluster.com>
+Anush Shetty <ashetty@redhat.com> <anush@gluster.com>
+Csaba Henk <csaba@redhat.com> <csaba@gluster.com> <csaba@lowlife.hu> <csaba@zresearch.com>
+Harshavardhana <fharshav@redhat.com> <harsha@gluster.com> <harsha@zresearch.com> <harsha@dev.gluster.com> <harsha@harshavardhana.net>
+Kaleb S. KEITHLEY <kkeithle@redhat.com> <kkeithle@f16node1.kkeithle.usersys.redhat.com>
+Kaushal M <kaushal@redhat.com> <kaushal@gluster.com>
+Kaushik BV <kbudiger@redhat.com> <kaushikbv@gluster.com>
+Krishna Srinivas <ksriniva@redhat.com> <krishna@gluster.com> <krishna@zresearch.com> <krishna@guest-laptop>
+Krishnan Parthasarathi <kparthas@redhat.com> <kp@gluster.com>
+Louis Zuckerman <louiszuckerman@gmail.com> <me@louiszuckerman.com>
+M S Vishwanath Bhat <vbhat@redhat.com> <msvbhat@gmail.com> <vishwanath@gluster.com>
+Pavan Sondur <pavan@gluster.com> <pavan@dev.gluster.com>
+Pete Zaitcev <zaitcev@kotori.zaitcev.us> <zaitcev@yahoo.com>
+Pranith Kumar K <pkarampu@redhat.com> <pranithk@gluster.com>
+Raghavendra Bhat <raghavendra@redhat.com> <raghavendrabhat@gluster.com>
+Raghavendra G <rgowdapp@redhat.com> <raghavendra@gluster.com> <raghavendra@zresearch.com>
+Rahul C S <rahulcs@redhat.com> <rahulcssjce@gmail.com>
+Rajesh Amaravathi <rajesh@redhat.com> <rajesh@gluster.com> <rajesh.amaravathi@gmail.com>
+Shehjar Tikoo <shehjart@gluster.com> <shehjart@zresearch.com>
+Venky Shankar <vshankar@redhat.com> <venky@gluster.com>
+Vijay Bellur <vbellur@redhat.com> <vijay@gluster.com> <vijay@dev.gluster.com>
+Vijaykumar Koppad <vkoppad@redhat.com> <vijaykumar.koppad@gmail.com>
+Vikas Gorur <vikas@gluster.com> <vikas@zresearch.com>
+shishir gowda <sgowda@redhat.com> <shishirng@gluster.com>
diff --git a/CONTRIBUTING b/CONTRIBUTING
index 8b3baa7e5..7bccd88d7 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -22,4 +22,4 @@
             are public and that a record of the contribution (including all
             personal information I submit with it, including my sign-off) is
             maintained indefinitely and may be redistributed consistent with
-            this project or the open source license(s) involved.
\ No newline at end of file
+            this project or the open source license(s) involved.
diff --git a/Makefile.am b/Makefile.am
index 836f17efc..598ebb410 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,13 +1,16 @@
 EXTRA_DIST = autogen.sh \
 	COPYING-GPLV2 COPYING-LGPLV3 \
 	INSTALL README AUTHORS THANKS NEWS \
-	glusterfs.spec glusterfs-api.pc.in
+	glusterfs.spec glusterfs-api.pc.in libgfchangelog.pc.in \
+	error-codes.json gf-error-codes.h.template \
+	gen-headers.py run-tests.sh \
+	$(shell find $(top_srcdir)/tests -type f -print)
 
 SUBDIRS = argp-standalone libglusterfs rpc api xlators glusterfsd \
-	$(FUSERMOUNT_SUBDIR) doc extras cli
+	$(FUSERMOUNT_SUBDIR) doc extras cli @SYNCDAEMON_SUBDIR@
 
 pkgconfigdir = @pkgconfigdir@
-pkgconfig_DATA = glusterfs-api.pc
+pkgconfig_DATA = glusterfs-api.pc libgfchangelog.pc
 
 CLEANFILES =
 
@@ -18,4 +21,9 @@ gitclean: distclean
 	rm -fr autom4te.cache
 	rm -f missing aclocal.m4 config.h.in config.guess config.sub ltmain.sh install-sh configure depcomp
 	rm -fr argp-standalone/autom4te.cache
-	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in argp-standalone/configure argp-standalone/depcomp argp-standalone/install-sh argp-standalone/missing
+	rm -f argp-standalone/aclocal.m4 argp-standalone/config.h.in
+	rm -f argp-standalone/configure argp-standalone/depcomp
+	rm -f argp-standalone/install-sh argp-standalone/missing
+
+dist-hook:
+	(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
diff --git a/api/Makefile.am b/api/Makefile.am
index af437a64d..f0ad1ee97 100644
--- a/api/Makefile.am
+++ b/api/Makefile.am
@@ -1 +1 @@
-SUBDIRS = src
+SUBDIRS = src examples
diff --git a/api/examples/Makefile.am b/api/examples/Makefile.am
index 6048bb1c8..05f40ff53 100644
--- a/api/examples/Makefile.am
+++ b/api/examples/Makefile.am
@@ -1,4 +1,6 @@
-noinst_PROGRAMS = glfsxmp
+EXTRA_PROGRAMS = glfsxmp
 glfsxmp_SOURCES = glfsxmp.c
 glfsxmp_CFLAGS = $(GLFS_CFLAGS) -Wall
-glfsxmp_LDADD = $(GLFS_LIBS)
\ No newline at end of file
+glfsxmp_LDADD = $(GLFS_LIBS) -lrt
+
+EXTRA_DIST = gfapi.py
diff --git a/api/examples/README b/api/examples/README
index 8f0bab697..4d2b521f7 100644
--- a/api/examples/README
+++ b/api/examples/README
@@ -17,3 +17,20 @@ install glusterfs-api RPM.
 Make sure your LDFLAGS includes -L/path/to/lib where libgfapi.so is
 installed and -I/path/to/include/glusterfs where the 'api' directory
 containing the headers are available.
+
+glfsxmp.c
+=========
+
+glfsxmp.c is an example application which uses libgfapi
+
+Compilation Steps For glfsxmp.c
+===============================
+
+1. $./autogen.sh
+2. $./configure
+
+Note: Before running ./configure , as mentioned above, you need to
+      take care of #1 or #2 i.e. pkg-config path or LDFLAGS and
+      -I/<path> with correct values.
+
+3. $make glfsxmp
diff --git a/api/examples/gfapi.py b/api/examples/gfapi.py
index e71a9866e..3ac67f4d5 100644..100755
--- a/api/examples/gfapi.py
+++ b/api/examples/gfapi.py
@@ -1,29 +1,422 @@
-import ctypes
+#!/usr/bin/python
+
+from ctypes import *
+from ctypes.util import find_library
 import os
 import sys
+import time
+import types
 
 # Looks like ctypes is having trouble with dependencies, so just force them to
 # load with RTLD_GLOBAL until I figure that out.
-glfs = ctypes.CDLL("libglusterfs.so",ctypes.RTLD_GLOBAL)
-xdr = ctypes.CDLL("libgfxdr.so",ctypes.RTLD_GLOBAL)
-api = ctypes.CDLL("api/libgfapi.so",ctypes.RTLD_GLOBAL)
-
-fs = api.glfs_new(sys.argv[1])
-api.glfs_set_logging(fs,"/dev/stderr",7)
-api.glfs_set_volfile_server(fs,"socket","localhost",24007)
-api.glfs_init(fs)
-print "Initialized volume"
-
-fd = api.glfs_creat(fs,sys.argv[2],os.O_RDWR,0644)
-print "Created file"
-
-# Read anything that's there from before.
-rbuf = ctypes.create_string_buffer(32)
-if api.glfs_read(fd,rbuf,32,0) > 0:
-	print "old data = %s" % rbuf.value
-
-# Write some new data.
-api.glfs_lseek(fd,0,os.SEEK_SET)
-wrote = api.glfs_write(fd,sys.argv[3],len(sys.argv[3]),0)
-if wrote > 0:
-	print "wrote %d bytes" % wrote
+glfs = CDLL(find_library("glusterfs"),RTLD_GLOBAL)
+xdr = CDLL(find_library("gfxdr"),RTLD_GLOBAL)
+api = CDLL(find_library("gfapi"),RTLD_GLOBAL)
+
+# Wow, the Linux kernel folks really play nasty games with this structure.  If
+# you look at the man page for stat(2) and then at this definition you'll note
+# two discrepancies.  First, we seem to have st_nlink and st_mode reversed.  In
+# fact that's exactly how they're defined *for 64-bit systems*; for 32-bit
+# they're in the man-page order.  Even uglier, the man page makes no mention of
+# the *nsec fields, but they are very much present and if they're not included
+# then we get memory corruption because libgfapi has a structure definition
+# that's longer than ours and they overwrite some random bit of memory after
+# the space we allocated.  Yes, that's all very disgusting, and I'm still not
+# sure this will really work on 32-bit because all of the field types are so
+# obfuscated behind macros and feature checks.
+class Stat (Structure):
+        _fields_ = [
+                ("st_dev",              c_ulong),
+                ("st_ino",              c_ulong),
+                ("st_nlink",            c_ulong),
+                ("st_mode",             c_uint),
+                ("st_uid",              c_uint),
+                ("st_gid",              c_uint),
+                ("st_rdev",             c_ulong),
+                ("st_size",             c_ulong),
+                ("st_blksize",          c_ulong),
+                ("st_blocks",           c_ulong),
+                ("st_atime",            c_ulong),
+                ("st_atimensec",        c_ulong),
+                ("st_mtime",            c_ulong),
+                ("st_mtimensec",        c_ulong),
+                ("st_ctime",            c_ulong),
+                ("st_ctimensec",        c_ulong),
+        ]
+api.glfs_creat.restype = c_void_p
+api.glfs_open.restype = c_void_p
+api.glfs_lstat.restype = c_int
+api.glfs_lstat.argtypes = [c_void_p, c_char_p, POINTER(Stat)]
+
+class Dirent (Structure):
+        _fields_ = [
+                ("d_ino",       c_ulong),
+                ("d_off",       c_ulong),
+                ("d_reclen",    c_ushort),
+                ("d_type",      c_char),
+                ("d_name",      c_char * 256),
+        ]
+api.glfs_opendir.restype = c_void_p
+api.glfs_readdir_r.restype = c_int
+api.glfs_readdir_r.argtypes = [c_void_p, POINTER(Dirent),
+                               POINTER(POINTER(Dirent))]
+
+# There's a bit of ctypes glitchiness around __del__ functions and module-level
+# variables.  If we unload the module while we still have references to File or
+# Volume objects, the module-level variables might have disappeared by the time
+# __del__ gets called.  Therefore the objects hold references which they
+# release when __del__ is done.  We only actually use the object-local values
+# in __del__; for clarity, we just use the simpler module-level form elsewhere.
+
+class File(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+
+        def __del__ (self):
+                self._api.glfs_close(self.fd)
+                self._api = None
+
+        # File operations, in alphabetical order.
+
+        def fsync (self):
+                return api.glfs_fsync(self.fd)
+
+        def read (self, buflen, flags=0):
+                rbuf = create_string_buffer(buflen)
+                rc = api.glfs_read(self.fd,rbuf,buflen,flags)
+                if rc > 0:
+                        return rbuf.value[:rc]
+                else:
+                        return rc
+
+        def read_buffer (self, buf, flags=0):
+                return api.glfs_read(self.fd,buf,len(buf),flags)
+
+        def write (self, data, flags=0):
+                return api.glfs_write(self.fd,data,len(data),flags)
+
+        def fallocate (self, mode, offset, len):
+            return api.glfs_fallocate(self.fd, mode, offset, len)
+
+        def discard (self, offset, len):
+            return api.glfs_discard(self.fd, offset, len)
+
+
+class Dir(object):
+
+        def __init__ (self, fd):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fd = fd
+                self.cursor = POINTER(Dirent)()
+
+        def __del__ (self):
+                self._api.glfs_closedir(self.fd)
+                self._api = None
+
+        def next (self):
+                entry = Dirent()
+                entry.d_reclen = 256
+                rc = api.glfs_readdir_r(self.fd,byref(entry),byref(self.cursor))
+                if (rc < 0) or (not self.cursor) or (not self.cursor.contents):
+                        return rc
+                return entry
+
+class Volume(object):
+
+        # Housekeeping functions.
+
+        def __init__ (self, host, volid, proto="tcp", port=24007):
+                # Add a reference so the module-level variable "api" doesn't
+                # get yanked out from under us (see comment above File def'n).
+                self._api = api
+                self.fs = api.glfs_new(volid)
+                api.glfs_set_volfile_server(self.fs,proto,host,port)
+
+        def __del__ (self):
+                self._api.glfs_fini(self.fs)
+                self._api = None
+
+        def set_logging (self, path, level):
+                api.glfs_set_logging(self.fs,path,level)
+
+        def mount (self):
+                api.glfs_init(self.fs)
+
+        # File operations, in alphabetical order.
+
+        def creat (self, path, flags, mode):
+                fd = api.glfs_creat(self.fs,path,flags,mode)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def getxattr (self, path, key, maxlen):
+                buf = create_string_buffer(maxlen)
+                rc = api.glfs_getxattr(self.fs,path,key,buf,maxlen)
+                if rc < 0:
+                        return rc
+                return buf.value[:rc]
+
+        def listxattr (self, path):
+                buf = create_string_buffer(512)
+                rc = api.glfs_listxattr(self.fs,path,buf,512)
+                if rc < 0:
+                        return rc
+                xattrs = []
+                # Parsing character by character is ugly, but it seems like the
+                # easiest way to deal with the "strings separated by NUL in one
+                # buffer" format.
+                i = 0
+                while i < rc:
+                        new_xa = buf.raw[i]
+                        i += 1
+                        while i < rc:
+                                next_char = buf.raw[i]
+                                i += 1
+                                if next_char == '\0':
+                                        xattrs.append(new_xa)
+                                        break
+                                new_xa += next_char
+                xattrs.sort()
+                return xattrs
+
+        def lstat (self, path):
+                x = Stat()
+                rc = api.glfs_lstat(self.fs,path,byref(x))
+                if rc >= 0:
+                        return x
+                else:
+                        return rc
+
+        def mkdir (self, path):
+                return api.glfs_mkdir(self.fs,path)
+
+        def open (self, path, flags):
+                fd = api.glfs_open(self.fs,path,flags)
+                if not fd:
+                        return fd
+                return File(fd)
+
+        def opendir (self, path):
+                fd = api.glfs_opendir(self.fs,path)
+                if not fd:
+                        return fd
+                return Dir(fd)
+
+        def rename (self, opath, npath):
+                return api.glfs_rename(self.fs,opath,npath)
+
+        def rmdir (self, path):
+                return api.glfs_rmdir(self.fs,path)
+
+        def setxattr (self, path, key, value, vlen):
+                return api.glfs_setxattr(self.fs,path,key,value,vlen,0)
+
+        def unlink (self, path):
+                return api.glfs_unlink(self.fs,path)
+
+if __name__ == "__main__":
+        def test_create_write (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+                rc = fd.write(data)
+                if rc != len(data):
+                        return False, "wrote %d/%d bytes" % (rc, len(data))
+                return True, "wrote %d bytes" % rc
+
+        # TBD: this test fails if we do create, open, write, read
+        def test_open_read (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.open(mypath,os.O_RDONLY)
+                if not fd:
+                        return False, "open error"
+                dlen = len(data) * 2
+                buf = fd.read(dlen)
+                if type(buf) == types.IntType:
+                        return False, "read error %d" % buf
+                if len(buf) != len(data):
+                        return False, "read %d/%d bytes" % (len(buf), len(data))
+                return True, "read '%s'" % buf
+
+        def test_lstat (vol, path, data):
+                mypath = path + ".io"
+                sb = vol.lstat(mypath)
+                if type(sb) == types.IntType:
+                        return False, "lstat error %d" % sb
+                if sb.st_size != len(data):
+                        return False, "lstat size is %d, expected %d" % (
+                                sb.st_size, len(data))
+                return True, "lstat got correct size %d" % sb.st_size
+
+        def test_rename (vol, path, data):
+                opath = path + ".io"
+                npath = path + ".tmp"
+                rc = vol.rename(opath,npath)
+                if rc < 0:
+                        return False, "rename error %d" % rc
+                ofd = vol.open(opath,os.O_RDWR)
+                if isinstance(ofd,File):
+                        return False, "old path working after rename"
+                nfd = vol.open(npath,os.O_RDWR)
+                if isinstance(nfd,File):
+                        return False, "new path not working after rename"
+                return True, "rename worked"
+
+        def test_unlink (vol, path, data):
+                mypath = path + ".tmp"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink error %d" % fd
+                fd = vol.open(mypath,os.O_RDWR)
+                if isinstance(fd,File):
+                        return False, "path still usable after unlink"
+                return True, "unlink worked"
+
+        def test_mkdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.mkdir(mypath)
+                if rc < 0:
+                        return False, "mkdir error %d" % rc
+                return True, "mkdir worked"
+
+        def test_create_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                fd = vol.creat(mypath,os.O_RDWR,0644)
+                if not isinstance(fd,File):
+                        return False, "create (in dir) error"
+                return True, "create (in dir) worked"
+
+        def test_dir_listing (vol, path, data):
+                mypath = path + ".dir"
+                fd = vol.opendir(mypath)
+                if not isinstance(fd,Dir):
+                        return False, "opendir error %d" % fd
+                files = []
+                while True:
+                        ent = fd.next()
+                        if not isinstance(ent,Dirent):
+                                break
+                        name = ent.d_name[:ent.d_reclen]
+                        files.append(name)
+                if files != [".", "..", "probe"]:
+                        return False, "wrong directory contents"
+                return True, "directory listing worked"
+
+        def test_unlink_in_dir (vol, path, data):
+                mypath = path + ".dir/probe"
+                rc = vol.unlink(mypath)
+                if rc < 0:
+                        return False, "unlink (in dir) error %d" % rc
+                return True, "unlink (in dir) worked"
+
+        def test_rmdir (vol, path, data):
+                mypath = path + ".dir"
+                rc = vol.rmdir(mypath)
+                if rc < 0:
+                        return False, "rmdir error %d" % rc
+                sb = vol.lstat(mypath)
+                if not isinstance(sb,Stat):
+                        return False, "dir still there after rmdir"
+                return True, "rmdir worked"
+
+        def test_setxattr (vol, path, data):
+                mypath = path + ".xa"
+                fd = vol.creat(mypath,os.O_RDWR|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat (xattr test) error"
+                key1, key2 = "hello", "goodbye"
+                if vol.setxattr(mypath,"trusted.key1",key1,len(key1)) < 0:
+                        return False, "setxattr (key1) error"
+                if vol.setxattr(mypath,"trusted.key2",key2,len(key2)) < 0:
+                        return False, "setxattr (key2) error"
+                return True, "setxattr worked"
+
+        def test_getxattr (vol, path, data):
+                mypath = path + ".xa"
+                buf = vol.getxattr(mypath,"trusted.key1",32)
+                if type(buf) == types.IntType:
+                        return False, "getxattr error"
+                if buf != "hello":
+                        return False, "wrong getxattr value %s" % buf
+                return True, "getxattr worked"
+
+        def test_listxattr (vol, path, data):
+                mypath = path + ".xa"
+                xattrs = vol.listxattr(mypath)
+                if type(xattrs) == types.IntType:
+                        return False, "listxattr error"
+                if xattrs != ["trusted.key1","trusted.key2"]:
+                        return False, "wrong listxattr value %s" % repr(xattrs)
+                return True, "listxattr worked"
+
+        def test_fallocate (vol, path, data):
+                mypath = path + ".io"
+                fd = vol.creat(mypath,os.O_WRONLY|os.O_EXCL,0644)
+                if not fd:
+                        return False, "creat error"
+		rc = fd.fallocate(0, 0, 1024*1024)
+                if rc != 0:
+                        return False, "fallocate error"
+		rc = fd.discard(4096, 4096)
+		if rc != 0:
+			return False, "discard error"
+                return True, "fallocate/discard worked"
+
+        test_list = (
+                test_create_write,
+                test_open_read,
+                test_lstat,
+                test_rename,
+                test_unlink,
+                test_mkdir,
+                test_create_in_dir,
+                test_dir_listing,
+                test_unlink_in_dir,
+                test_rmdir,
+                test_setxattr,
+                test_getxattr,
+                test_listxattr,
+                test_fallocate,
+        )
+
+        ok_to_fail = (
+                # TBD: this fails opening the new file, even though the file
+                # did get renamed.  Looks like a gfapi bug, not ours.
+                (test_rename, "new path not working after rename"),
+                # TBD: similar, call returns error even though it worked
+                (test_rmdir, "dir still there after rmdir"),
+        )
+
+        volid, path = sys.argv[1:3]
+        data = "fubar"
+        vol = Volume("localhost",volid)
+        vol.set_logging("/dev/null",7)
+        #vol.set_logging("/dev/stderr",7)
+        vol.mount()
+
+        failures = 0
+        expected = 0
+        for t in test_list:
+                rc, msg = t(vol,path,data)
+                if rc:
+                        print "PASS: %s" % msg
+                else:
+                        print "FAIL: %s" % msg
+                        failures += 1
+                        for otf in ok_to_fail:
+                                if (t == otf[0]) and (msg == otf[1]):
+                                        print "  (skipping known failure)"
+                                        expected += 1
+                                        break # from the *inner* for loop
+                        else:
+                                break # from the *outer* for loop
+
+        print "%d failures (%d expected)" % (failures, expected)
diff --git a/api/examples/glfsxmp.c b/api/examples/glfsxmp.c
index 9497128af..600d72fb5 100644
--- a/api/examples/glfsxmp.c
+++ b/api/examples/glfsxmp.c
@@ -1,83 +1,1598 @@
 #include <stdio.h>
+#include <stdlib.h>
 #include <errno.h>
 #include "api/glfs.h"
+#include "api/glfs-handles.h"
 #include <string.h>
 #include <time.h>
 
+
+int
+test_dirops (glfs_t *fs)
+{
+        glfs_fd_t *fd = NULL;
+        char buf[512];
+        struct dirent *entry = NULL;
+
+        fd = glfs_opendir (fs, "/");
+        if (!fd) {
+                fprintf (stderr, "/: %s\n", strerror (errno));
+                return -1;
+        }
+
+        fprintf (stderr, "Entries:\n");
+        while (glfs_readdir_r (fd, (struct dirent *)buf, &entry), entry) {
+                fprintf (stderr, "%s: %lu\n", entry->d_name, glfs_telldir (fd));
+        }
+
+        glfs_closedir (fd);
+        return 0;
+}
+
+
+int
+test_xattr (glfs_t *fs)
+{
+        char *filename = "/filename2";
+        char buf[512];
+        char *ptr;
+        int ret;
+
+        ret = glfs_setxattr (fs, filename, "user.testkey", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_setxattr (fs, filename, "user.testkey2", "testval", 8, 0);
+        fprintf (stderr, "setxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+
+        ret = glfs_listxattr (fs, filename, buf, 512);
+        fprintf (stderr, "listxattr(%s): %d (%s)\n", filename, ret,
+                 strerror (errno));
+        if (ret < 0)
+                return -1;
+
+        for (ptr = buf; ptr < buf + ret; ptr++) {
+                printf ("key=%s\n", ptr);
+                ptr += strlen (ptr);
+        }
+
+        return 0;
+}
+
+
+int
+test_chdir (glfs_t *fs)
+{
+        int ret = -1;
+        char *topdir = "/topdir";
+        char *linkdir = "/linkdir";
+        char *subdir = "./subdir";
+        char *respath = NULL;
+        char pathbuf[4096];
+
+        ret = glfs_mkdir (fs, topdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", topdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        ret = glfs_symlink (fs, topdir, linkdir);
+        if (ret) {
+                fprintf (stderr, "symlink(%s, %s): %s\n", topdir, linkdir, strerror (errno));
+                return -1;
+        }
+
+        ret = glfs_chdir (fs, linkdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", linkdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (respath) {
+                fprintf (stderr, "realpath(%s) worked unexpectedly: %s\n", subdir, respath);
+                return -1;
+        }
+
+        ret = glfs_mkdir (fs, subdir, 0755);
+        if (ret) {
+                fprintf (stderr, "mkdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_realpath (fs, subdir, pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(%s): %s\n", subdir, strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(%s) = %s\n", subdir, respath);
+        }
+
+        ret = glfs_chdir (fs, subdir);
+        if (ret) {
+                fprintf (stderr, "chdir(%s): %s\n", subdir, strerror (errno));
+                return -1;
+        }
+
+        respath = glfs_getcwd (fs, pathbuf, 4096);
+        fprintf (stdout, "getcwd() = %s\n", respath);
+
+        respath = glfs_realpath (fs, "/linkdir/subdir", pathbuf);
+        if (!respath) {
+                fprintf (stderr, "realpath(/linkdir/subdir): %s\n", strerror (errno));
+        } else {
+                fprintf (stdout, "realpath(/linkdir/subdir) = %s\n", respath);
+        }
+
+        return 0;
+}
+
+#ifdef DEBUG
+static void
+peek_stat (struct stat *sb)
+{
+        printf ("Dumping stat information:\n");
+        printf ("File type:                ");
+
+        switch (sb->st_mode & S_IFMT) {
+                case S_IFBLK:  printf ("block device\n");            break;
+                case S_IFCHR:  printf ("character device\n");        break;
+                case S_IFDIR:  printf ("directory\n");               break;
+                case S_IFIFO:  printf ("FIFO/pipe\n");               break;
+                case S_IFLNK:  printf ("symlink\n");                 break;
+                case S_IFREG:  printf ("regular file\n");            break;
+                case S_IFSOCK: printf ("socket\n");                  break;
+                default:       printf ("unknown?\n");                break;
+        }
+
+        printf ("I-node number:            %ld\n", (long) sb->st_ino);
+
+        printf ("Mode:                     %lo (octal)\n",
+                (unsigned long) sb->st_mode);
+
+        printf ("Link count:               %ld\n", (long) sb->st_nlink);
+        printf ("Ownership:                UID=%ld   GID=%ld\n",
+                (long) sb->st_uid, (long) sb->st_gid);
+
+        printf ("Preferred I/O block size: %ld bytes\n",
+                (long) sb->st_blksize);
+        printf ("File size:                %lld bytes\n",
+                (long long) sb->st_size);
+        printf ("Blocks allocated:         %lld\n",
+                (long long) sb->st_blocks);
+
+        printf ("Last status change:       %s", ctime(&sb->st_ctime));
+        printf ("Last file access:         %s", ctime(&sb->st_atime));
+        printf ("Last file modification:   %s", ctime(&sb->st_mtime));
+
+        return;
+}
+
+static void
+peek_handle (unsigned char *glid)
+{
+        int i;
+
+        for (i = 0; i < GFAPI_HANDLE_LENGTH; i++)
+        {
+                printf (":%02x:", glid[i]);
+        }
+        printf ("\n");
+}
+#else /* DEBUG */
+static void
+peek_stat (struct stat *sb)
+{
+        return;
+}
+
+static void
+peek_handle (unsigned char *id)
+{
+        return;
+}
+#endif /* DEBUG */
+
+glfs_t    *fs = NULL;
+char      *full_parent_name = "/testdir", *parent_name = "testdir";
+
+void
+test_h_unlink (void)
+{
+        char               *my_dir = "unlinkdir";
+        char               *my_file = "file.txt";
+        char               *my_subdir = "dir1";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL,
+                           *subdir = NULL, *subleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf ("glfs_h_unlink tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subdir = glfs_h_mkdir (fs, dir, my_subdir, 0644, &sb);
+        if (subdir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        subleaf = glfs_h_creat (fs, subdir, my_file, O_CREAT, 0644, &sb);
+        if (subleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non empty directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOTEMPTY) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: it is non empty: %s\n",
+                         my_subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, subdir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, subdir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_subdir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant regular file */
+        ret = glfs_h_unlink (fs, dir, my_file);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s: invalid errno ,%d, %s\n",
+                         my_file, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink non-existant directory */
+        ret = glfs_h_unlink (fs, dir, my_subdir);
+        if ((ret && errno != ENOENT) || (ret == 0)) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking non-existant %s:  invalid errno ,%d, %s\n",
+                         my_subdir, ret, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        /* unlink directory */
+        ret = glfs_h_unlink (fs, parent, my_dir);
+        if (ret) {
+                fprintf (stderr, "glfs_h_unlink: error unlinking %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_unlink tests: PASSED\n");
+
+out:
+        if (dir)
+                glfs_h_close (dir);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (subdir)
+                glfs_h_close (subdir);
+        if (subleaf)
+                glfs_h_close (subleaf);
+        if (parent)
+                glfs_h_close (parent);
+
+        return;
+}
+
+void
+test_h_getsetattrs (void)
+{
+        char               *my_dir = "attrdir";
+        char               *my_file = "attrfile.txt";
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb, retsb;
+        int                 ret, valid;
+        struct timespec     timestamp;
+
+        printf("glfs_h_getattrs and setattrs tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, dir, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dir, strerror (errno));
+                printf ("glfs_h_unlink tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_getattrs (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+        /* TODO: Compare stat information */
+
+        retsb.st_mode = 00666;
+        retsb.st_uid = 1000;
+        retsb.st_gid = 1001;
+        ret = clock_gettime (CLOCK_REALTIME, &timestamp);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        retsb.st_atim = timestamp;
+        retsb.st_mtim = timestamp;
+        valid = GFAPI_SET_ATTR_MODE | GFAPI_SET_ATTR_UID | GFAPI_SET_ATTR_GID |
+        GFAPI_SET_ATTR_ATIME | GFAPI_SET_ATTR_MTIME;
+        peek_stat (&retsb);
+
+        ret = glfs_h_setattrs (fs, dir, &retsb, valid);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_setattrs: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        memset(&retsb, 0, sizeof (struct stat));
+        ret = glfs_h_stat (fs, dir, &retsb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_stat: error %s: from (%p),%s\n",
+                         my_dir, dir, strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&retsb);
+
+        printf ("glfs_h_getattrs and setattrs tests: PASSED\n");
+out:
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dir)
+                glfs_h_close (dir);
+
+        return;
+}
+
+void
+test_h_truncate (void)
+{
+        char               *my_dir = "truncatedir";
+        char               *my_file = "file.txt";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL;
+        struct stat         sb;
+        glfs_fd_t          *fd = NULL;
+        char                buf[32];
+        off_t               offset = 0;
+        int                 ret = 0;
+
+        printf("glfs_h_truncate tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         my_file, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        memcpy (buf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, buf, 32, 0);
+
+        /* run tests */
+        /* truncate lower */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate higher */
+        offset = 32;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        /* truncate equal */
+        offset = 30;
+        ret = glfs_h_truncate (fs, leaf, offset);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_truncate: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        ret = glfs_h_getattrs (fs, leaf, &sb);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_getattrs: error for %s (%p),%s\n",
+                         my_file, leaf, strerror (errno));
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+        if (sb.st_size != offset) {
+                fprintf (stderr, "glfs_h_truncate: post size mismatch\n");
+                printf ("glfs_h_truncate tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_truncate tests: PASSED\n");
+out:
+        if (fd)
+                glfs_close (fd);
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return;
+}
+
+void
+test_h_links (void)
+{
+        char               *my_dir = "linkdir";
+        char               *my_file = "file.txt";
+        char               *my_symlnk = "slnk.txt";
+        char               *my_lnk = "lnk.txt";
+        char               *linksrc_dir = "dir1";
+        char               *linktgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct glfs_object *ln1 = NULL;
+        struct stat         sb;
+        int                 ret;
+        char               *buf = NULL;
+
+        printf("glfs_h_link(s) tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, linksrc_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linksrc_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, linktgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         linktgt_dir, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* sym link: /testdir/linkdir/file.txt to ./slnk.txt */
+        ln1 = glfs_h_symlink (fs, parent, my_symlnk, "./file.txt", &sb);
+        if (ln1 == NULL) {
+                fprintf (stderr, "glfs_h_symlink: error creating %s: from (%p),%s\n",
+                         my_symlnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        buf = calloc (1024, sizeof(char));
+        if (buf == NULL) {
+                fprintf (stderr, "Error allocating memory\n");
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        ret = glfs_h_readlink (fs, ln1, buf, 1024);
+        if (ret <= 0) {
+                fprintf (stderr, "glfs_h_readlink: error reading %s: from (%p),%s\n",
+                         my_symlnk, ln1, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        if (!(strncmp (buf, my_symlnk, strlen (my_symlnk)))) {
+                fprintf (stderr, "glfs_h_readlink: error mismatch in link name: actual %s: retrieved %s\n",
+                         my_symlnk, buf);
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+
+        /* link: /testdir/linkdir/file.txt to ./lnk.txt */
+        ret = glfs_h_link (fs, leaf, parent, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, parent, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        /* link: /testdir/linkdir/dir1/file.txt to ../dir2/slnk.txt */
+        ret = glfs_h_link (fs, dleaf, dirtgt, my_lnk);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_link: error creating %s: from (%p),%s\n",
+                         my_lnk, dirtgt, strerror (errno));
+                printf ("glfs_h_link(s) tests: FAILED\n");
+                goto out;
+        }
+        /* TODO: Should write content to a file and read from the link */
+
+        printf ("glfs_h_link(s) tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+        if (ln1)
+                glfs_h_close (ln1);
+        if (buf)
+                free (buf);
+
+        return;
+}
+
+void
+test_h_rename (void)
+{
+        char               *my_dir = "renamedir";
+        char               *my_file = "file.txt";
+        char               *src_dir = "dir1";
+        char               *tgt_dir = "dir2";
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *dirsrc = NULL, *dirtgt = NULL, *dleaf = NULL;
+        struct stat         sb;
+        int                 ret;
+
+        printf("glfs_h_rename tests: In Progress\n");
+
+        /* Prepare tests */
+        root = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_mkdir (fs, root, my_dir, 0644, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, root, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, my_file, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirsrc = glfs_h_mkdir (fs, parent, src_dir, 0644, &sb);
+        if (dirsrc == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         src_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dirtgt = glfs_h_mkdir (fs, parent, tgt_dir, 0644, &sb);
+        if (dirtgt == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         tgt_dir, parent, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        dleaf = glfs_h_creat (fs, dirsrc, my_file, O_CREAT, 0644, &sb);
+        if (dleaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                         my_file, dirsrc, strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* run tests */
+        /* Rename file.txt -> file1.txt */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "file1.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file1.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1/file.txt -> file.txt */
+        ret = glfs_h_rename (fs, dirsrc, "file.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s/%s to %s (%s)\n",
+                         src_dir, "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file1.txt -> file.txt (exists) */
+        ret = glfs_h_rename (fs, parent, "file1.txt", parent, "file.txt");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "file.txt", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir1 -> dir3 */
+        ret = glfs_h_rename (fs, parent, "dir1", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir1", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir2 ->dir3 (exists) */
+        ret = glfs_h_rename (fs, parent, "dir2", parent, "dir3");
+        if (ret != 0) {
+                fprintf (stderr, "glfs_h_rename: error renaming %s to %s (%s)\n",
+                         "dir2", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename file.txt -> dir3 (fail) */
+        ret = glfs_h_rename (fs, parent, "file.txt", parent, "dir3");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "file.txt", "dir3", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        /* rename dir3 -> file.txt (fail) */
+        ret = glfs_h_rename (fs, parent, "dir3", parent, "file.txt");
+        if (ret == 0) {
+                fprintf (stderr, "glfs_h_rename: NO error renaming %s to %s (%s)\n",
+                         "dir3", "file.txt", strerror (errno));
+                printf ("glfs_h_rename tests: FAILED\n");
+                goto out;
+        }
+
+        printf ("glfs_h_rename tests: PASSED\n");
+
+out:
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+        if (dirsrc)
+                glfs_h_close (dirsrc);
+        if (dirtgt)
+                glfs_h_close (dirtgt);
+        if (dleaf)
+                glfs_h_close (dleaf);
+
+        return;
+}
+
+void
+assimilatetime (struct timespec *ts, struct timespec ts_st,
+                struct timespec ts_ed)
+{
+        if ((ts_ed.tv_nsec - ts_st.tv_nsec) < 0) {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec - 1;
+                ts->tv_nsec += 1000000000 + ts_ed.tv_nsec - ts_st.tv_nsec;
+        } else {
+                ts->tv_sec += ts_ed.tv_sec - ts_st.tv_sec;
+                ts->tv_nsec += ts_ed.tv_nsec - ts_st.tv_nsec;
+        }
+
+        if (ts->tv_nsec > 1000000000) {
+                ts->tv_nsec = ts->tv_nsec - 1000000000;
+                ts->tv_sec += 1;
+        }
+
+        return;
+}
+
+#define MAX_FILES_CREATE 10
+#define MAXPATHNAME      512
+void
+test_h_performance (void)
+{
+        char               *my_dir = "perftest",
+                           *full_dir_path="/testdir/perftest";
+        char               *my_file = "file_", my_file_name[MAXPATHNAME];
+        struct glfs_object *parent = NULL, *leaf = NULL, *dir = NULL;
+        struct stat         sb;
+        int                 ret, i;
+        struct glfs_fd     *fd;
+        struct timespec     c_ts = {0, 0}, c_ts_st, c_ts_ed;
+        struct timespec     o_ts = {0, 0}, o_ts_st, o_ts_ed;
+
+        printf("glfs_h_performance tests: In Progress\n");
+
+        /* Prepare tests */
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, NULL, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+
+        dir = glfs_h_mkdir (fs, parent, my_dir, 0644, &sb);
+        if (dir == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* create performance */
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s%d", my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_lookupat (fs, dir, my_file_name, &sb);
+                if (leaf != NULL) {
+                        fprintf (stderr, "glfs_h_lookup: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                leaf = glfs_h_creat (fs, dir, my_file_name, O_CREAT, 0644, &sb);
+                if (leaf == NULL) {
+                        fprintf (stderr, "glfs_h_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_h_close (leaf); leaf = NULL;
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (handle based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+
+        /* create using path */
+        c_ts.tv_sec = o_ts.tv_sec = 0;
+        c_ts.tv_nsec = o_ts.tv_nsec = 0;
+
+        sprintf (my_file_name, "%s1", full_dir_path);
+        ret = glfs_mkdir (fs, my_file_name, 0644);
+        if (ret != 0) {
+                fprintf (stderr, "glfs_mkdir: error creating %s: from (%p),%s\n",
+                         my_dir, parent, strerror (errno));
+                printf ("glfs_h_performance tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_st);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        for (i = 0; i < MAX_FILES_CREATE; i++) {
+                sprintf (my_file_name, "%s1/%sn%d", full_dir_path, my_file, i);
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_st);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = glfs_stat (fs, my_file_name, &sb);
+                if (ret == 0) {
+                        fprintf (stderr, "glfs_stat: exists %s\n",
+                                 my_file_name);
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                fd = glfs_creat (fs, my_file_name, O_CREAT, 0644);
+                if (fd == NULL) {
+                        fprintf (stderr, "glfs_creat: error creating %s: from (%p),%s\n",
+                                 my_file, dir, strerror (errno));
+                        printf ("glfs_h_performance tests: FAILED\n");
+                        goto out;
+                }
+
+                ret = clock_gettime (CLOCK_REALTIME, &c_ts_ed);
+                if(ret != 0) {
+                        fprintf (stderr, "clock_gettime: error %s\n",
+                                 strerror (errno));
+                        printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                        goto out;
+                }
+
+                assimilatetime (&c_ts, c_ts_st, c_ts_ed);
+                glfs_close (fd);
+        }
+
+        ret = clock_gettime (CLOCK_REALTIME, &o_ts_ed);
+        if(ret != 0) {
+                fprintf (stderr, "clock_gettime: error %s\n", strerror (errno));
+                printf ("glfs_h_getattrs and setattrs tests: FAILED\n");
+                goto out;
+        }
+
+        assimilatetime (&o_ts, o_ts_st, o_ts_ed);
+
+        printf ("Creation performance (path based):\n\t# empty files:%d\n",
+                MAX_FILES_CREATE);
+        printf ("\tOverall time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                o_ts.tv_sec, o_ts.tv_nsec);
+        printf ("\tcreate call time time:\n\t\tSecs:%ld\n\t\tnSecs:%ld\n",
+                c_ts.tv_sec, c_ts.tv_nsec);
+out:
+        return;
+}
+
+int
+test_handleops (int argc, char *argv[])
+{
+        int                 ret = 0;
+        glfs_fd_t          *fd = NULL;
+        struct stat         sb = {0, };
+        struct glfs_object *root = NULL, *parent = NULL, *leaf = NULL,
+                           *tmp = NULL;
+        char                readbuf[32], writebuf[32];
+        unsigned char       leaf_handle[GFAPI_HANDLE_LENGTH];
+
+        char *full_leaf_name = "/testdir/testfile.txt",
+             *leaf_name = "testfile.txt",
+             *relative_leaf_name = "testdir/testfile.txt";
+        char *leaf_name1 = "testfile1.txt";
+        char *full_newparent_name = "/testdir/dir1",
+             *newparent_name = "dir1";
+        char *full_newnod_name = "/testdir/nod1",
+             *newnod_name = "nod1";
+
+        /* Initialize test area */
+        ret = glfs_mkdir (fs, full_parent_name, 0644);
+        if (ret != 0 && errno != EEXIST) {
+                fprintf (stderr, "%s: (%p) %s\n", full_parent_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        else if (ret != 0) {
+                printf ("Found test directory %s to be existing\n",
+                        full_parent_name);
+                printf ("Cleanup test directory and restart tests\n");
+                goto out;
+        }
+
+        fd = glfs_creat (fs, full_leaf_name, O_CREAT, 0644);
+        if (fd == NULL) {
+                fprintf (stderr, "%s: (%p) %s\n", full_leaf_name, fd,
+                        strerror (errno));
+                printf ("Test initialization failed on volume %s\n", argv[1]);
+                goto out;
+        }
+        glfs_close (fd);
+
+        printf ("Initialized the test area, within volume %s\n", argv[1]);
+
+        /* Handle based APIs test area */
+
+        /* glfs_lookupat test */
+        printf ("glfs_h_lookupat tests: In Progress\n");
+        /* start at root of the volume */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a parent within root */
+        parent = glfs_h_lookupat (fs, root, parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* lookup a leaf/child within the parent */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check absolute paths */
+        root = glfs_h_lookupat (fs, NULL, "/", &sb);
+        if (root == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         "/", NULL, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_lookupat (fs, NULL, full_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_leaf_name, parent, strerror (errno));
+                printf ("glfs_h_lookupat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (leaf); leaf = NULL;
+
+        /* check multiple component paths */
+        leaf = glfs_h_lookupat (fs, root, relative_leaf_name, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         relative_leaf_name, parent, strerror (errno));
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* reset */
+        glfs_h_close (root); root = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        /* check symlinks in path */
+
+        /* TODO: -ve test cases */
+        /* parent invalid
+        * path invalid
+        * path does not exist after some components
+        * no parent, but relative path
+        * parent and full path? -ve?
+        */
+
+        printf ("glfs_h_lookupat tests: PASSED\n");
+
+        /* glfs_openat test */
+        printf ("glfs_h_open tests: In Progress\n");
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", readbuf,
+                        writebuf);
+                glfs_close (fd);
+                printf ("glfs_h_open tests: FAILED\n");
+                goto out;
+        }
+
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_close (fd);
+
+        printf ("glfs_h_open tests: PASSED\n");
+
+        /* Create tests */
+        printf ("glfs_h_creat tests: In Progress\n");
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_creat: error on create of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_creat (fs, parent, leaf_name1, O_CREAT | O_EXCL, 0644,
+                            &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_creat: existing file, leaf = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        tmp = glfs_h_creat (fs, root, parent_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || !(errno == EISDIR || errno == EINVAL)) {
+                fprintf (stderr, "glfs_h_creat: dir create, tmp = (%p), errno = %s\n",
+                        leaf, strerror (errno));
+                printf ("glfs_h_creat tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close (tmp); tmp = NULL;
+                }
+        }
+
+        /* TODO: Other combinations and -ve cases as applicable */
+        printf ("glfs_h_creat tests: PASSED\n");
+
+        /* extract handle and create from handle test */
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: In Progress\n");
+        /* TODO: Change the lookup to creat below for a GIFD recovery falure,
+         * that needs to be fixed */
+        leaf = glfs_h_lookupat (fs, parent, leaf_name1, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         leaf_name1, parent, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        ret = glfs_h_extract_handle (leaf, leaf_handle,
+                                             GFAPI_HANDLE_LENGTH);
+        if (ret < 0) {
+                fprintf (stderr, "glfs_h_extract_handle: error extracting handle of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_extract_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_handle (leaf_handle);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_create_from_handle (fs, leaf_handle, GFAPI_HANDLE_LENGTH,
+                                          &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_create_from_handle: error on create of %s: from (%p),%s\n",
+                         leaf_name1, leaf_handle, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        fd = glfs_h_open (fs, leaf, O_RDWR);
+        if (fd == NULL) {
+                fprintf (stderr, "glfs_h_open: error on open of %s: %s\n",
+                         full_leaf_name, strerror (errno));
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                goto out;
+        }
+
+        /* test read/write based on fd */
+        memcpy (writebuf, "abcdefghijklmnopqrstuvwxyz012345", 32);
+        ret = glfs_write (fd, writebuf, 32, 0);
+
+        glfs_lseek (fd, 0, SEEK_SET);
+
+        ret = glfs_read (fd, readbuf, 32, 0);
+        if (memcmp (readbuf, writebuf, 32)) {
+                printf ("Failed to read what I wrote: %s %s\n", writebuf,
+                        writebuf);
+                printf ("glfs_h_create_from_handle tests: FAILED\n");
+                glfs_close (fd);
+                goto out;
+        }
+
+        glfs_close (fd);
+        glfs_h_close (leaf); leaf = NULL;
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_extract_handle and glfs_h_create_from_handle tests: PASSED\n");
+
+        /* Mkdir tests */
+        printf ("glfs_h_mkdir tests: In Progress\n");
+
+        ret = glfs_rmdir (fs, full_newparent_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_rmdir: Failed for %s: %s\n",
+                         full_newparent_name, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newparent_name, parent, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mkdir (fs, parent, newparent_name, 0644, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mkdir: existing directory, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mkdir tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mkdir tests: PASSED\n");
+
+        /* Mknod tests */
+        printf ("glfs_h_mknod tests: In Progress\n");
+        ret = glfs_unlink (fs, full_newnod_name);
+        if (ret && errno != ENOENT) {
+                fprintf (stderr, "glfs_unlink: Failed for %s: %s\n",
+                         full_newnod_name, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+
+        parent = glfs_h_lookupat (fs, NULL, full_parent_name, &sb);
+        if (parent == NULL) {
+                fprintf (stderr, "glfs_h_lookupat: error on lookup of %s: from (%p),%s\n",
+                         full_parent_name, root, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, S_IFIFO, 0, &sb);
+        if (leaf == NULL) {
+                fprintf (stderr, "glfs_h_mkdir: error on mkdir of %s: from (%p),%s\n",
+                         newnod_name, parent, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                goto out;
+        }
+        peek_stat (&sb);
+
+        /* TODO: creat op on a FIFO node hangs, need to check and fix
+        tmp = glfs_h_creat (fs, parent, newnod_name, O_CREAT, 0644, &sb);
+        if (tmp != NULL || errno != EINVAL) {
+                fprintf (stderr, "glfs_h_creat: node create, tmp = (%p), errno = %s\n",
+                        tmp, strerror (errno));
+                printf ("glfs_h_creat/mknod tests: FAILED\n");
+                if (tmp != NULL) {
+                        glfs_h_close(tmp); tmp = NULL;
+                }
+        } */
+
+        glfs_h_close (leaf); leaf = NULL;
+
+        leaf = glfs_h_mknod (fs, parent, newnod_name, 0644, 0, &sb);
+        if (leaf != NULL || errno != EEXIST) {
+                fprintf (stderr, "glfs_h_mknod: existing node, leaf = (%p), errno = %s\n",
+                         leaf, strerror (errno));
+                printf ("glfs_h_mknod tests: FAILED\n");
+                if (leaf != NULL) {
+                        glfs_h_close (leaf); leaf = NULL;
+                }
+        }
+
+        glfs_h_close (parent); parent = NULL;
+
+        printf ("glfs_h_mknod tests: PASSED\n");
+
+        /* unlink tests */
+        test_h_unlink ();
+
+        /* TODO: opendir tests */
+
+        /* getattr tests */
+        test_h_getsetattrs ();
+
+        /* TODO: setattr tests */
+
+        /* truncate tests */
+        test_h_truncate();
+
+        /* link tests */
+        test_h_links ();
+
+        /* rename tests */
+        test_h_rename ();
+
+        /* performance tests */
+        test_h_performance ();
+
+        /* END: New APIs test area */
+
+out:
+        /* Cleanup glfs handles */
+        if (root)
+                glfs_h_close (root);
+        if (parent)
+                glfs_h_close (parent);
+        if (leaf)
+                glfs_h_close (leaf);
+
+        return ret;
+}
+
 int
 main (int argc, char *argv[])
 {
-	glfs_t    *fs = NULL;
-	glfs_t    *fs2 = NULL;
-	int        ret = 0;
-	glfs_fd_t *fd = NULL;
-	glfs_fd_t *fd2 = NULL;
-	struct stat sb = {0, };
-	char       readbuf[32];
-	char       writebuf[32];
+        glfs_t    *fs2 = NULL;
+        int        ret = 0;
+        glfs_fd_t *fd = NULL;
+        glfs_fd_t *fd2 = NULL;
+        struct stat sb = {0, };
+        char       readbuf[32];
+        char       writebuf[32];
+
+        char      *filename = "/filename2";
+
+        if (argc != 3) {
+                printf ("Expect following args\n\t%s <volname> <hostname>\n", argv[0]);
+                return -1;
+        }
+
+        fs = glfs_new (argv[1]);
+        if (!fs) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+//      ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+
+        ret = glfs_set_volfile_server (fs, "tcp", argv[2], 24007);
+
+//      ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+
+        ret = glfs_set_logging (fs, "/dev/stderr", 7);
+
+        ret = glfs_init (fs);
+
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
+
+        sleep (2);
+
+        fs2 = glfs_new (argv[1]);
+        if (!fs2) {
+                fprintf (stderr, "glfs_new: returned NULL\n");
+                return 1;
+        }
+
+
+//      ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
 
-	char      *filename = "/filename2";
+        ret = glfs_set_volfile_server (fs2, "tcp", argv[2], 24007);
 
-	fs = glfs_new ("iops");
-	if (!fs) {
-		fprintf (stderr, "glfs_new: returned NULL\n");
-		return 1;
-	}
+        ret = glfs_set_logging (fs2, "/dev/stderr", 7);
 
-//	ret = glfs_set_volfile (fs, "/tmp/posix.vol");
+        ret = glfs_init (fs2);
 
-	ret = glfs_set_volfile_server (fs, "socket", "localhost", 24007);
+        fprintf (stderr, "glfs_init: returned %d\n", ret);
 
-//	ret = glfs_set_volfile_server (fs, "unix", "/tmp/gluster.sock", 0);
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
 
-	ret = glfs_set_logging (fs, "/dev/stderr", 7);
+        fd = glfs_creat (fs, filename, O_RDWR, 0644);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
 
-	ret = glfs_init (fs);
+        fd2 = glfs_open (fs2, filename, O_RDWR);
+        fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
 
-	fprintf (stderr, "glfs_init: returned %d\n", ret);
+        sprintf (writebuf, "hi there\n");
+        ret = glfs_write (fd, writebuf, 32, 0);
 
-	sleep (2);
+        glfs_lseek (fd2, 0, SEEK_SET);
 
-	fs2 = glfs_new ("iops");
-	if (!fs2) {
-		fprintf (stderr, "glfs_new: returned NULL\n");
-		return 1;
-	}
+        ret = glfs_read (fd2, readbuf, 32, 0);
 
+        printf ("read %d, %s", ret, readbuf);
 
-//	ret = glfs_set_volfile (fs2, "/tmp/posix.vol");
+        glfs_close (fd);
+        glfs_close (fd2);
 
-	ret = glfs_set_volfile_server (fs2, "socket", "localhost", 24007);
+        filename = "/filename3";
+        ret = glfs_mknod (fs, filename, S_IFIFO, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
 
-	ret = glfs_set_logging (fs2, "/dev/stderr", 7);
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
 
-	ret = glfs_init (fs2);
 
-	fprintf (stderr, "glfs_init: returned %d\n", ret);
+        ret = glfs_rename (fs, filename, "/filename4");
+        fprintf (stderr, "rename(%s): (%d) %s\n", filename, ret,
+                 strerror (errno));
 
-	ret = glfs_lstat (fs, filename, &sb);
-	fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
+        ret = glfs_unlink (fs, "/filename4");
+        fprintf (stderr, "unlink(%s): (%d) %s\n", "/filename4", ret,
+                 strerror (errno));
 
-	fd = glfs_creat (fs, filename, O_RDWR, 0644);
-	fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+        filename = "/dirname2";
+        ret = glfs_mkdir (fs, filename, 0);
+        fprintf (stderr, "%s: (%d) %s\n", filename, ret, strerror (errno));
 
-	fd2 = glfs_open (fs2, filename, O_RDWR);
-	fprintf (stderr, "%s: (%p) %s\n", filename, fd, strerror (errno));
+        ret = glfs_lstat (fs, filename, &sb);
+        fprintf (stderr, "lstat(%s): (%d) %s\n", filename, ret, strerror (errno));
 
-	sprintf (writebuf, "hi there\n");
-	ret = glfs_write (fd, writebuf, 32, 0);
+        ret = glfs_rmdir (fs, filename);
+        fprintf (stderr, "rmdir(%s): (%d) %s\n", filename, ret, strerror (errno));
 
-	glfs_lseek (fd2, 0, SEEK_SET);
+        test_dirops (fs);
 
-	ret = glfs_read (fd2, readbuf, 32, 0);
+        test_xattr (fs);
 
-	printf ("read %d, %s", ret, readbuf);
+        test_chdir (fs);
 
-	glfs_close (fd);
-	glfs_close (fd2);
+        test_handleops (argc, argv);
+        // done
 
-	glfs_fini (fs);
-	glfs_fini (fs2);
+        glfs_fini (fs);
+        glfs_fini (fs2);
 
-	return ret;
+        return ret;
 }
diff --git a/api/examples/setup.py.in b/api/examples/setup.py.in
new file mode 100644
index 000000000..44b738094
--- /dev/null
+++ b/api/examples/setup.py.in
@@ -0,0 +1,29 @@
+from distutils.core import setup
+
+# generate a __init__.py for the package namespace
+fo = open('__init__.py', 'w')
+fo.write('__version__ = "@PACKAGE_VERSION@"\n')
+fo.close()
+
+DESC = """GlusterFS is a clustered file-system capable of scaling to
+several petabytes. It aggregates various storage bricks over Infiniband
+RDMA or TCP/IP interconnect into one large parallel network file system.
+GlusterFS is one of the most sophisticated file systems in terms of
+features and extensibility.  It borrows a powerful concept called
+Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
+user space and easily manageable.
+
+This package contains the Python interface to the libgfapi library."""
+
+setup(
+    name='glusterfs-api',
+    version='@PACKAGE_VERSION@',
+    description='Python client library for the GlusterFS libgfapi',
+    long_description=DESC,
+    author='Gluster Community',
+    author_email='gluster-devel@nongnu.org',
+    license='LGPLv3',
+    url='http://gluster.org/',
+    package_dir={'gluster':''},
+    packages=['gluster']
+)
diff --git a/api/src/Makefile.am b/api/src/Makefile.am
index f5fb9e70b..7c5df3e20 100644
--- a/api/src/Makefile.am
+++ b/api/src/Makefile.am
@@ -1,30 +1,36 @@
 lib_LTLIBRARIES = libgfapi.la
 noinst_HEADERS = glfs-mem-types.h glfs-internal.h
-libgfapi_HEADERS = glfs.h
+libgfapi_HEADERS = glfs.h glfs-handles.h
 libgfapidir = $(includedir)/glusterfs/api
 
-libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c
-libgfapi_la_CFLAGS = -fPIC -Wall
-libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+libgfapi_la_SOURCES = glfs.c glfs-mgmt.c glfs-fops.c glfs-resolve.c \
+	glfs-handleops.c
+libgfapi_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(GF_LDADD)
 
-libgfapi_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE \
-	-D$(GF_HOST_OS) $(GF_CFLAGS) \
+libgfapi_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \
 	-I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src \
 	-I$(top_srcdir)/rpc/xdr/src
-libgfapi_la_LDFLAGS = -shared -nostartfiles
 
 
 xlator_LTLIBRARIES = api.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
+# workaround for broken parallel install support in automake with LTLIBRARIES
+# http://debbugs.gnu.org/cgi/bugreport.cgi?bug=7328
+install_xlatorLTLIBRARIES = install-xlatorLTLIBRARIES
+$(install_xlatorLTLIBRARIES): install-libLTLIBRARIES
 
 api_la_SOURCES = glfs-master.c
+api_la_DEPENDENCIES = libgfapi.la
+api_la_LDFLAGS = -module -avoid-version
+api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/api/src/libgfapi.la
 
-api_la_LDFLAGS = -module -avoidversion
-api_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) -Wall \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index f3fba84e4..10bb7d38b 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -13,6 +13,56 @@
 #include "glfs-mem-types.h"
 #include "syncop.h"
 #include "glfs.h"
+#include <limits.h>
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
+
+#define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
+
+int
+glfs_loc_link (loc_t *loc, struct iatt *iatt)
+{
+	int ret = -1;
+	inode_t *linked_inode = NULL;
+
+	if (!loc->inode) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	linked_inode = inode_link (loc->inode, loc->parent, loc->name, iatt);
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
+		ret = 0;
+	} else {
+		ret = -1;
+		errno = ENOMEM;
+	}
+
+	return ret;
+}
+
+
+void
+glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat)
+{
+	iatt_to_stat (iatt, stat);
+	stat->st_dev = fs->dev_id;
+}
+
+
+int
+glfs_loc_unlink (loc_t *loc)
+{
+	inode_unlink (loc->inode, loc->parent, loc->name);
+
+	return 0;
+}
 
 
 struct glfs_fd *
@@ -23,6 +73,7 @@ glfs_open (struct glfs *fs, const char *path, int flags)
 	xlator_t        *subvol = NULL;
 	loc_t            loc = {0, };
 	struct iatt      iatt = {0, };
+	int              reval = 0;
 
 	__glfs_entry_fs (fs);
 
@@ -33,11 +84,15 @@ glfs_open (struct glfs *fs, const char *path, int flags)
 		goto out;
 	}
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
+	glfd = glfs_fd_new (fs);
 	if (!glfd)
 		goto out;
 
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt);
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
 	if (ret)
 		goto out;
 
@@ -53,6 +108,14 @@ glfs_open (struct glfs *fs, const char *path, int flags)
 		goto out;
 	}
 
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
 	glfd->fd = fd_create (loc.inode, getpid());
 	if (!glfd->fd) {
 		ret = -1;
@@ -61,14 +124,22 @@ glfs_open (struct glfs *fs, const char *path, int flags)
 	}
 
 	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
 out:
 	loc_wipe (&loc);
 
 	if (ret && glfd) {
 		glfs_fd_destroy (glfd);
 		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
 	}
 
+	glfs_subvol_done (fs, subvol);
+
 	return glfd;
 }
 
@@ -78,15 +149,35 @@ glfs_close (struct glfs_fd *glfd)
 {
 	xlator_t  *subvol = NULL;
 	int        ret = -1;
+	fd_t      *fd = NULL;
+	struct glfs *fs = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                ret = -1;
+                errno = EIO;
+                goto out;
+        }
 
-	ret = syncop_flush (subvol, glfd->fd);
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
 
+	ret = syncop_flush (subvol, fd);
+out:
+	fs = glfd->fs;
 	glfs_fd_destroy (glfd);
 
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (fs, subvol);
+
 	return ret;
 }
 
@@ -98,6 +189,40 @@ glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
 	xlator_t        *subvol = NULL;
 	loc_t            loc = {0, };
 	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0 && stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_stat (struct glfs *fs, const char *path, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
 
 	__glfs_entry_fs (fs);
 
@@ -107,14 +232,18 @@ glfs_lstat (struct glfs *fs, const char *path, struct stat *stat)
 		errno = EIO;
 		goto out;
 	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
 
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt);
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
 
 	if (ret == 0 && stat)
-		iatt_to_stat (&iatt, stat);
+		glfs_iatt_to_stat (fs, &iatt, stat);
 out:
 	loc_wipe (&loc);
 
+	glfs_subvol_done (fs, subvol);
+
 	return ret;
 }
 
@@ -125,21 +254,34 @@ glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
 	struct iatt      iatt = {0, };
+	fd_t            *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
 		errno = EIO;
 		goto out;
 	}
 
-	ret = syncop_fstat (subvol, glfd->fd, &iatt);
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fstat (subvol, fd, &iatt);
 
 	if (ret == 0 && stat)
-		iatt_to_stat (&iatt, stat);
+		glfs_iatt_to_stat (glfd->fs, &iatt, stat);
 out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
 	return ret;
 }
 
@@ -154,6 +296,7 @@ glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
 	struct iatt      iatt = {0, };
 	uuid_t           gfid;
 	dict_t          *xattr_req = NULL;
+	int              reval = 0;
 
 	__glfs_entry_fs (fs);
 
@@ -179,11 +322,20 @@ glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
 		goto out;
 	}
 
-	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
+	glfd = glfs_fd_new (fs);
 	if (!glfd)
 		goto out;
 
-	ret = glfs_resolve (fs, subvol, path, &loc, &iatt);
+	/* This must be glfs_resolve() and NOT glfs_lresolve().
+	   That is because open("name", O_CREAT) where "name"
+	   is a danging symlink must create the dangling
+	   destinataion.
+	*/
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
 	if (ret == -1 && errno != ENOENT)
 		/* Any other type of error is fatal */
 		goto out;
@@ -223,6 +375,14 @@ glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
 		}
 	}
 
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
 	glfd->fd = fd_create (loc.inode, getpid());
 	if (!glfd->fd) {
 		ret = -1;
@@ -230,18 +390,34 @@ glfs_creat (struct glfs *fs, const char *path, int flags, mode_t mode)
 		goto out;
 	}
 
-	ret = syncop_create (subvol, &loc, flags, mode, glfd->fd, xattr_req);
+	if (ret == 0) {
+		ret = syncop_open (subvol, &loc, flags, glfd->fd);
+	} else {
+		ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+				     xattr_req, &iatt);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
 out:
 	loc_wipe (&loc);
 
 	if (xattr_req)
-		dict_destroy (xattr_req);
+		dict_unref (xattr_req);
 
 	if (ret && glfd) {
 		glfs_fd_destroy (glfd);
 		glfd = NULL;
+	} else if (glfd) {
+                glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
 	}
 
+	glfs_subvol_done (fs, subvol);
+
 	return glfd;
 }
 
@@ -282,33 +458,52 @@ glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
 	     off_t offset, int flags)
 {
 	xlator_t       *subvol = NULL;
-	int             ret = -1;
-	size_t          size = -1;
+	ssize_t         ret = -1;
+	ssize_t         size = -1;
 	struct iovec   *iov = NULL;
 	int             cnt = 0;
 	struct iobref  *iobref = NULL;
+	fd_t           *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
 
 	size = iov_length (iovec, iovcnt);
 
-	ret = syncop_readv (subvol, glfd->fd, size, offset,
-			    0, &iov, &cnt, &iobref);
+	ret = syncop_readv (subvol, fd, size, offset, 0, &iov, &cnt, &iobref);
 	if (ret <= 0)
-		return ret;
+		goto out;
 
 	size = iov_copy (iovec, iovcnt, iov, cnt); /* FIXME!!! */
 
 	glfd->offset = (offset + size);
 
-	if (iov)
-		GF_FREE (iov);
-	if (iobref)
-		iobref_unref (iobref);
+	ret = size;
+out:
+        if (iov)
+                GF_FREE (iov);
+        if (iobref)
+                iobref_unref (iobref);
 
-	return size;
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
 }
 
 
@@ -388,10 +583,6 @@ glfs_io_async_task (void *data)
 	ssize_t         ret = 0;
 
 	switch (gio->op) {
-	case GF_FOP_READ:
-		ret = glfs_preadv (gio->glfd, gio->iov, gio->count,
-				   gio->offset, gio->flags);
-		break;
 	case GF_FOP_WRITE:
 		ret = glfs_pwritev (gio->glfd, gio->iov, gio->count,
 				    gio->offset, gio->flags);
@@ -405,6 +596,12 @@ glfs_io_async_task (void *data)
 		else
 			ret = glfs_fsync (gio->glfd);
 		break;
+	case GF_FOP_DISCARD:
+		ret = glfs_discard (gio->glfd, gio->offset, gio->count);
+		break;
+        case GF_FOP_ZEROFILL:
+                ret = glfs_zerofill(gio->glfd, gio->offset, gio->count);
+                break;
 	}
 
 	return (int) ret;
@@ -412,23 +609,90 @@ glfs_io_async_task (void *data)
 
 
 int
+glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, struct iovec *iovec,
+		       int count, struct iatt *stbuf, struct iobref *iobref,
+		       dict_t *xdata)
+{
+	struct glfs_io *gio = NULL;
+	xlator_t       *subvol = NULL;
+	struct glfs    *fs = NULL;
+	struct glfs_fd *glfd = NULL;
+
+
+	gio = frame->local;
+	frame->local = NULL;
+	subvol = cookie;
+	glfd = gio->glfd;
+	fs = glfd->fs;
+
+	if (op_ret <= 0)
+		goto out;
+
+	op_ret = iov_copy (gio->iov, gio->count, iovec, count);
+
+	glfd->offset = gio->offset + op_ret;
+out:
+	errno = op_errno;
+	gio->fn (gio->glfd, op_ret, gio->data);
+
+	GF_FREE (gio->iov);
+	GF_FREE (gio);
+	STACK_DESTROY (frame->root);
+	glfs_subvol_done (fs, subvol);
+
+	return 0;
+}
+
+
+int
 glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
 		   off_t offset, int flags, glfs_io_cbk fn, void *data)
 {
 	struct glfs_io *gio = NULL;
 	int             ret = 0;
+	call_frame_t   *frame = NULL;
+	xlator_t       *subvol = NULL;
+	glfs_t         *fs = NULL;
+	fd_t           *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	fs = glfd->fs;
+
+	frame = syncop_create_frame (THIS);
+	if (!frame) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
 
 	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
 	if (!gio) {
+		ret = -1;
 		errno = ENOMEM;
-		return -1;
+		goto out;
 	}
 
 	gio->iov = iov_dup (iovec, count);
 	if (!gio->iov) {
-		GF_FREE (gio);
+		ret = -1;
 		errno = ENOMEM;
-		return -1;
+		goto out;
 	}
 
 	gio->op     = GF_FOP_READ;
@@ -439,15 +703,23 @@ glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec, int count,
 	gio->fn     = fn;
 	gio->data   = data;
 
-	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
-			    glfs_io_async_task, glfs_io_async_cbk,
-			    NULL, gio);
+	frame->local = gio;
+
+	STACK_WIND_COOKIE (frame, glfs_preadv_async_cbk, subvol, subvol,
+			   subvol->fops->readv, fd, iov_length (iovec, count),
+			   offset, flags, NULL);
 
+out:
 	if (ret) {
 		GF_FREE (gio->iov);
 		GF_FREE (gio);
+		STACK_DESTROY (frame->root);
+		glfs_subvol_done (fs, subvol);
 	}
 
+	if (fd)
+		fd_unref (fd);
+
 	return ret;
 }
 
@@ -507,24 +779,39 @@ glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
 	struct iobref  *iobref = NULL;
 	struct iobuf   *iobuf = NULL;
 	struct iovec    iov = {0, };
+	fd_t           *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
 
 	size = iov_length (iovec, iovcnt);
 
 	iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
 	if (!iobuf) {
+		ret = -1;
 		errno = ENOMEM;
-		return -1;
+		goto out;
 	}
 
 	iobref = iobref_new ();
 	if (!iobref) {
 		iobuf_unref (iobuf);
 		errno = ENOMEM;
-		return -1;
+		ret = -1;
+		goto out;
 	}
 
 	ret = iobref_add (iobref, iobuf);
@@ -532,7 +819,8 @@ glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
 		iobuf_unref (iobuf);
 		iobref_unref (iobref);
 		errno = ENOMEM;
-		return -1;
+		ret = -1;
+		goto out;
 	}
 
 	iov_unload (iobuf_ptr (iobuf), iovec, iovcnt);  /* FIXME!!! */
@@ -540,17 +828,22 @@ glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
 	iov.iov_base = iobuf_ptr (iobuf);
 	iov.iov_len = size;
 
-	ret = syncop_writev (subvol, glfd->fd, &iov, 1, offset,
-			     iobref, flags);
+	ret = syncop_writev (subvol, fd, &iov, 1, offset, iobref, flags);
 
 	iobuf_unref (iobuf);
 	iobref_unref (iobref);
 
 	if (ret <= 0)
-		return ret;
+		goto out;
 
 	glfd->offset = (offset + size);
 
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
 	return ret;
 }
 
@@ -689,19 +982,31 @@ glfs_fsync (struct glfs_fd *glfd)
 {
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
 		errno = EIO;
 		goto out;
 	}
 
-	ret = syncop_fsync (subvol, glfd->fd);
-//	ret = syncop_fsync (subvol, glfd->fd, 0);
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 0);
 out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
 	return ret;
 }
 
@@ -751,19 +1056,31 @@ glfs_fdatasync (struct glfs_fd *glfd)
 {
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
 		errno = EIO;
 		goto out;
 	}
 
-	ret = syncop_fsync (subvol, glfd->fd);
-//	ret = syncop_fsync (subvol, glfd->fd, 1);
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsync (subvol, fd, 1);
 out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
 	return ret;
 }
 
@@ -780,18 +1097,31 @@ glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
 {
 	int              ret = -1;
 	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
 
 	__glfs_entry_fd (glfd);
 
-	subvol = glfs_fd_subvol (glfd);
+	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
 		errno = EIO;
 		goto out;
 	}
 
-	ret = syncop_ftruncate (subvol, glfd->fd, offset);
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_ftruncate (subvol, fd, offset);
 out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
 	return ret;
 }
 
@@ -827,3 +1157,2096 @@ glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset,
 	return ret;
 }
 
+
+int
+glfs_access (struct glfs *fs, const char *path, int mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_access (subvol, &loc, mode);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_symlink (struct glfs *fs, const char *data, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_readlink (struct glfs *fs, const char *path, char *buf, size_t bufsiz)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+	char            *linkval = NULL;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFLNK) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+	if (ret > 0) {
+		memcpy (buf, linkval, ret);
+		GF_FREE (linkval);
+	}
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mknod (struct glfs *fs, const char *path, mode_t mode, dev_t dev)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_mkdir (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	uuid_t           gfid;
+	dict_t          *xattr_req = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (loc.inode) {
+		errno = EEXIST;
+		ret = -1;
+		goto out;
+	}
+
+	if (ret == -1 && errno != ENOENT)
+		/* Any other type of error is fatal */
+		goto out;
+
+	if (ret == -1 && errno == ENOENT && !loc.parent)
+		/* The parent directory or an ancestor even
+		   higher does not exist
+		*/
+		goto out;
+
+	/* ret == -1 && errno == ENOENT */
+	loc.inode = inode_new (loc.parent->table);
+	if (!loc.inode) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_link (&loc, &iatt);
+out:
+	loc_wipe (&loc);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_unlink (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	ret = syncop_unlink (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rmdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (iatt.ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	ret = syncop_rmdir (subvol, &loc);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_rename (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+                if ((oldiatt.ia_type == IA_IFDIR) !=
+                    (newiatt.ia_type == IA_IFDIR)) {
+                       /* Either both old and new must be dirs,
+                        * or both must be non-dirs. Else, fail.
+                        */
+                       ret = -1;
+                       errno = EISDIR;
+                       goto out;
+                }
+        }
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		if (reval < DEFAULT_REVAL_COUNT) {
+			reval++;
+			loc_wipe (&oldloc);
+			loc_wipe (&newloc);
+			goto retry;
+		}
+	}
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_link (struct glfs *fs, const char *oldpath, const char *newpath)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            oldloc = {0, };
+	loc_t            newloc = {0, };
+	struct iatt      oldiatt = {0, };
+	struct iatt      newiatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_lresolve (fs, subvol, oldpath, &oldloc, &oldiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &oldloc, retry);
+
+	if (ret)
+		goto out;
+retrynew:
+	ret = glfs_lresolve (fs, subvol, newpath, &newloc, &newiatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &newloc, retrynew);
+
+	if (ret == 0) {
+		ret = -1;
+		errno = EEXIST;
+		goto out;
+	}
+
+	if (oldiatt.ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+        /* Filling the inode of the hard link to be same as that of the
+           original file
+        */
+	if (newloc.inode) {
+		inode_unref (newloc.inode);
+		newloc.inode = NULL;
+	}
+        newloc.inode = inode_ref (oldloc.inode);
+
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == -1 && errno == ESTALE) {
+		loc_wipe (&oldloc);
+		loc_wipe (&newloc);
+		if (reval--)
+			goto retry;
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_link (&newloc, &oldiatt);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_opendir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	if (glfd->fd) {
+		/* Retry. Safe to touch glfd->fd as we
+		   still have not glfs_fd_bind() yet.
+		*/
+		fd_unref (glfd->fd);
+		glfd->fd = NULL;
+	}
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+
+int
+glfs_closedir (struct glfs_fd *glfd)
+{
+	__glfs_entry_fd (glfd);
+
+	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
+
+	glfs_fd_destroy (glfd);
+
+	return 0;
+}
+
+
+long
+glfs_telldir (struct glfs_fd *fd)
+{
+	return fd->offset;
+}
+
+
+void
+glfs_seekdir (struct glfs_fd *fd, long offset)
+{
+	gf_dirent_t *entry = NULL;
+	gf_dirent_t *tmp = NULL;
+
+	if (fd->offset == offset)
+		return;
+
+	fd->offset = offset;
+	fd->next = NULL;
+
+	list_for_each_entry_safe (entry, tmp, &fd->entries, list) {
+		if (entry->d_off != offset)
+			continue;
+
+		if (&tmp->list != &fd->entries) {
+			/* found! */
+			fd->next = tmp;
+			return;
+		}
+	}
+	/* could not find entry at requested offset in the cache.
+	   next readdir_r() will result in glfd_entry_refresh()
+	*/
+}
+
+int
+glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
+		      glfs_io_cbk fn, void *data)
+{
+	struct glfs_io *gio = NULL;
+	int             ret = 0;
+
+	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+	if (!gio) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	gio->op     = GF_FOP_DISCARD;
+	gio->glfd   = glfd;
+	gio->offset = offset;
+	gio->count  = len;
+	gio->fn     = fn;
+	gio->data   = data;
+
+	ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+			    glfs_io_async_task, glfs_io_async_cbk,
+			    NULL, gio);
+
+	if (ret) {
+		GF_FREE (gio->iov);
+		GF_FREE (gio);
+	}
+
+	return ret;
+}
+
+int
+glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, size_t len,
+                      glfs_io_cbk fn, void *data)
+{
+        struct glfs_io *gio  = NULL;
+        int             ret  = 0;
+
+        gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
+        if (!gio) {
+                errno = ENOMEM;
+                return -1;
+        }
+
+        gio->op     = GF_FOP_ZEROFILL;
+        gio->glfd   = glfd;
+        gio->offset = offset;
+        gio->count  = len;
+        gio->fn     = fn;
+        gio->data   = data;
+
+        ret = synctask_new (glfs_from_glfd (glfd)->ctx->env,
+                            glfs_io_async_task, glfs_io_async_cbk,
+                            NULL, gio);
+
+        if (ret) {
+                GF_FREE (gio->iov);
+                GF_FREE (gio);
+        }
+
+        return ret;
+}
+
+
+void
+gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent)
+{
+	dirent->d_ino = gf_dirent->d_ino;
+
+#ifdef _DIRENT_HAVE_D_OFF
+	dirent->d_off = gf_dirent->d_off;
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+	dirent->d_type = gf_dirent->d_type;
+#endif
+
+#ifdef _DIRENT_HAVE_D_NAMLEN
+	dirent->d_namlen = strlen (gf_dirent->d_name);
+#endif
+
+	strncpy (dirent->d_name, gf_dirent->d_name, GF_NAME_MAX + 1);
+}
+
+
+int
+glfd_entry_refresh (struct glfs_fd *glfd, int plus)
+{
+	xlator_t        *subvol = NULL;
+	gf_dirent_t      entries;
+	gf_dirent_t      old;
+	int              ret = -1;
+	fd_t            *fd = NULL;
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	if (fd->inode->ia_type != IA_IFDIR) {
+		ret = -1;
+		errno = EBADF;
+		goto out;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+	INIT_LIST_HEAD (&old.list);
+
+	if (plus)
+		ret = syncop_readdirp (subvol, fd, 131072, glfd->offset,
+				       NULL, &entries);
+	else
+		ret = syncop_readdir (subvol, fd, 131072, glfd->offset,
+				      &entries);
+	if (ret >= 0) {
+		if (plus)
+			gf_link_inodes_from_dirent (THIS, fd->inode, &entries);
+
+		list_splice_init (&glfd->entries, &old.list);
+		list_splice_init (&entries.list, &glfd->entries);
+
+		/* spurious errno is dangerous for glfd_entry_next() */
+		errno = 0;
+	}
+
+	if (ret > 0)
+		glfd->next = list_entry (glfd->entries.next, gf_dirent_t, list);
+
+	gf_dirent_free (&old);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+gf_dirent_t *
+glfd_entry_next (struct glfs_fd *glfd, int plus)
+{
+	gf_dirent_t     *entry = NULL;
+	int              ret = -1;
+
+	if (!glfd->offset || !glfd->next) {
+		ret = glfd_entry_refresh (glfd, plus);
+		if (ret < 0)
+			return NULL;
+	}
+
+	entry = glfd->next;
+	if (!entry)
+		return NULL;
+
+	if (&entry->next->list == &glfd->entries)
+		glfd->next = NULL;
+	else
+		glfd->next = entry->next;
+
+	glfd->offset = entry->d_off;
+
+	return entry;
+}
+
+
+static struct dirent *
+glfs_readdirbuf_get (struct glfs_fd *glfd)
+{
+        struct dirent *buf = NULL;
+
+        LOCK (&glfd->fd->lock);
+        {
+                buf = glfd->readdirbuf;
+                if (buf) {
+                        memset (buf, 0, READDIRBUF_SIZE);
+                        goto unlock;
+                }
+
+                buf = GF_CALLOC (1, READDIRBUF_SIZE, glfs_mt_readdirbuf_t);
+                if (!buf) {
+                        errno = ENOMEM;
+                        goto unlock;
+                }
+
+                glfd->readdirbuf = buf;
+        }
+unlock:
+        UNLOCK (&glfd->fd->lock);
+
+        return buf;
+}
+
+
+int
+glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat, struct dirent *ext,
+		    struct dirent **res)
+{
+	int              ret = 0;
+	gf_dirent_t     *entry = NULL;
+	struct dirent   *buf = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	errno = 0;
+
+	if (ext)
+		buf = ext;
+	else
+		buf = glfs_readdirbuf_get (glfd);
+
+	if (!buf) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	entry = glfd_entry_next (glfd, !!stat);
+	if (errno)
+		ret = -1;
+
+	if (res) {
+		if (entry)
+			*res = buf;
+		else
+			*res = NULL;
+	}
+
+	if (entry) {
+		gf_dirent_to_dirent (entry, buf);
+		if (stat)
+			glfs_iatt_to_stat (glfd->fs, &entry->d_stat, stat);
+	}
+
+	return ret;
+}
+
+
+int
+glfs_readdir_r (struct glfs_fd *glfd, struct dirent *buf, struct dirent **res)
+{
+	return glfs_readdirplus_r (glfd, 0, buf, res);
+}
+
+
+struct dirent *
+glfs_readdirplus (struct glfs_fd *glfd, struct stat *stat)
+{
+        struct dirent *res = NULL;
+        int ret = -1;
+
+        ret = glfs_readdirplus_r (glfd, stat, NULL, &res);
+        if (ret)
+                return NULL;
+
+        return res;
+}
+
+
+
+struct dirent *
+glfs_readdir (struct glfs_fd *glfd)
+{
+        return glfs_readdirplus (glfd, NULL);
+}
+
+
+int
+glfs_statvfs (struct glfs *fs, const char *path, struct statvfs *buf)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_statfs (subvol, &loc, buf);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setattr (struct glfs *fs, const char *path, struct iatt *iatt,
+	      int valid, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      riatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &riatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &riatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_setattr (subvol, &loc, iatt, valid, 0, 0);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fsetattr (subvol, fd, iatt, valid, 0, 0);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_chmod (struct glfs *fs, const char *path, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_fchmod (struct glfs_fd *glfd, mode_t mode)
+{
+	int              ret = -1;
+	struct iatt      iatt = {0, };
+	int              valid = 0;
+
+	iatt.ia_prot = ia_prot_from_st_mode (mode);
+	valid = GF_SET_ATTR_MODE;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_chown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lchown (struct glfs *fs, const char *path, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_fchown (struct glfs_fd *glfd, uid_t uid, gid_t gid)
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_uid = uid;
+	iatt.ia_gid = gid;
+	valid = GF_SET_ATTR_UID|GF_SET_ATTR_GID;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_utimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 1);
+
+	return ret;
+}
+
+
+int
+glfs_lutimens (struct glfs *fs, const char *path, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_setattr (fs, path, &iatt, valid, 0);
+
+	return ret;
+}
+
+
+int
+glfs_futimens (struct glfs_fd *glfd, struct timespec times[2])
+{
+	int              ret = -1;
+	int              valid = 0;
+	struct iatt      iatt = {0, };
+
+	iatt.ia_atime = times[0].tv_sec;
+	iatt.ia_atime_nsec = times[0].tv_nsec;
+	iatt.ia_mtime = times[1].tv_sec;
+	iatt.ia_mtime_nsec = times[1].tv_nsec;
+
+	valid = GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME;
+
+	ret = glfs_fsetattr (glfd, &iatt, valid);
+
+	return ret;
+}
+
+
+int
+glfs_getxattr_process (void *value, size_t size, dict_t *xattr,
+		       const char *name)
+{
+	data_t *data = NULL;
+	int     ret = -1;
+
+	data = dict_get (xattr, (char *)name);
+	if (!data) {
+		errno = ENODATA;
+		ret = -1;
+		goto out;
+	}
+
+	ret = data->len;
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (value, data->data, ret);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr_common (struct glfs *fs, const char *path, const char *name,
+		      void *value, size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_getxattr (struct glfs *fs, const char *path, const char *name,
+	       void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 1);
+}
+
+
+ssize_t
+glfs_lgetxattr (struct glfs *fs, const char *path, const char *name,
+		void *value, size_t size)
+{
+	return glfs_getxattr_common (fs, path, name, value, size, 0);
+}
+
+
+ssize_t
+glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
+		size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, name);
+	if (ret)
+		goto out;
+
+	ret = glfs_getxattr_process (value, size, xattr, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_listxattr_process (void *value, size_t size, dict_t *xattr)
+{
+	int     ret = -1;
+
+	ret = dict_keys_join (NULL, 0, xattr, NULL);
+
+	if (!value || !size)
+		goto out;
+
+	if (size < ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	dict_keys_join (value, size, xattr, NULL);
+out:
+	if (xattr)
+		dict_unref (xattr);
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr_common (struct glfs *fs, const char *path, void *value,
+		       size_t size, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, NULL);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+ssize_t
+glfs_listxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 1);
+}
+
+
+ssize_t
+glfs_llistxattr (struct glfs *fs, const char *path, void *value, size_t size)
+{
+	return glfs_listxattr_common (fs, path, value, size, 0);
+}
+
+
+ssize_t
+glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fgetxattr (subvol, fd, &xattr, NULL);
+	if (ret)
+		goto out;
+
+	ret = glfs_listxattr_process (value, size, xattr);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+dict_t *
+dict_for_key_value (const char *name, const char *value, size_t size)
+{
+	dict_t *xattr = NULL;
+	int     ret = 0;
+
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
+
+	ret = dict_set_static_bin (xattr, (char *)name, (void *)value, size);
+	if (ret) {
+		dict_destroy (xattr);
+		xattr = NULL;
+	}
+
+	return xattr;
+}
+
+
+int
+glfs_setxattr_common (struct glfs *fs, const char *path, const char *name,
+		      const void *value, size_t size, int flags, int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	dict_t          *xattr = NULL;
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_setxattr (subvol, &loc, xattr, flags);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_setxattr (struct glfs *fs, const char *path, const char *name,
+	       const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 1);
+}
+
+
+int
+glfs_lsetxattr (struct glfs *fs, const char *path, const char *name,
+		const void *value, size_t size, int flags)
+{
+	return glfs_setxattr_common (fs, path, name, value, size, flags, 0);
+}
+
+
+int
+glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
+		size_t size, int flags)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	dict_t          *xattr = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	xattr = dict_for_key_value (name, value, size);
+	if (!xattr) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (subvol, fd, xattr, flags);
+out:
+	if (xattr)
+		dict_unref (xattr);
+
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr_common (struct glfs *fs, const char *path, const char *name,
+			 int follow)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	if (follow)
+		ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+	else
+		ret = glfs_lresolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	ret = syncop_removexattr (subvol, &loc, name);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_removexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 1);
+}
+
+
+int
+glfs_lremovexattr (struct glfs *fs, const char *path, const char *name)
+{
+	return glfs_removexattr_common (fs, path, name, 0);
+}
+
+
+int
+glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fremovexattr (subvol, fd, name);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_fallocate (subvol, fd, keep_size, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	fd_t		*fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	ret = syncop_discard (subvol, fd, offset, len);
+out:
+	if (fd)
+		fd_unref(fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_zerofill (struct glfs_fd *glfd, off_t offset, size_t len)
+{
+        int               ret             = -1;
+        xlator_t         *subvol          = NULL;
+        fd_t             *fd              = NULL;
+
+        __glfs_entry_fd (glfd);
+
+        subvol = glfs_active_subvol (glfd->fs);
+        if (!subvol) {
+                errno = EIO;
+                goto out;
+        }
+
+        fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+        if (!fd) {
+                errno = EBADFD;
+                goto out;
+        }
+
+        ret = syncop_zerofill (subvol, fd, offset, len);
+out:
+        if (fd)
+                fd_unref(fd);
+
+        glfs_subvol_done (glfd->fs, subvol);
+
+        return ret;
+}
+
+int
+glfs_chdir (struct glfs *fs, const char *path)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (!IA_ISDIR (iatt.ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (fs, loc.inode);
+
+out:
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+
+int
+glfs_fchdir (struct glfs_fd *glfd)
+{
+	int       ret = -1;
+	inode_t  *inode = NULL;
+	xlator_t *subvol = NULL;
+	fd_t     *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	inode = fd->inode;
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfs_cwd_set (glfd->fs, inode);
+	ret = 0;
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+char *
+glfs_realpath (struct glfs *fs, const char *path, char *resolved_path)
+{
+	int              ret = -1;
+	char            *retpath = NULL;
+	char            *allocpath = NULL;
+	xlator_t        *subvol = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              reval = 0;
+
+	__glfs_entry_fs (fs);
+
+	if (resolved_path)
+		retpath = resolved_path;
+	else
+		retpath = allocpath = malloc (PATH_MAX + 1);
+
+	if (!retpath) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+retry:
+	ret = glfs_resolve (fs, subvol, path, &loc, &iatt, reval);
+
+	ESTALE_RETRY (ret, errno, reval, &loc, retry);
+
+	if (ret)
+		goto out;
+
+	if (loc.path) {
+		strncpy (retpath, loc.path, PATH_MAX);
+		retpath[PATH_MAX] = 0;
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (ret == -1) {
+		if (allocpath)
+			free (allocpath);
+		retpath = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return retpath;
+}
+
+
+char *
+glfs_getcwd (struct glfs *fs, char *buf, size_t n)
+{
+	int              ret = -1;
+	inode_t         *inode = NULL;
+	char            *path = NULL;
+
+	__glfs_entry_fs (fs);
+
+	if (!buf || n < 2) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	inode = glfs_cwd_get (fs);
+
+	if (!inode) {
+		strncpy (buf, "/", n);
+		ret = 0;
+		goto out;
+	}
+
+	ret = inode_path (inode, 0, &path);
+	if (n <= ret) {
+		ret = -1;
+		errno = ERANGE;
+		goto out;
+	}
+
+	strncpy (buf, path, n);
+	ret = 0;
+out:
+	GF_FREE (path);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret < 0)
+		return NULL;
+
+	return buf;
+}
+
+
+static void
+gf_flock_to_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	flock->l_type   = gf_flock->l_type;
+	flock->l_whence = gf_flock->l_whence;
+	flock->l_start  = gf_flock->l_start;
+	flock->l_len    = gf_flock->l_len;
+	flock->l_pid    = gf_flock->l_pid;
+}
+
+
+static void
+gf_flock_from_flock (struct gf_flock *gf_flock, struct flock *flock)
+{
+	gf_flock->l_type   = flock->l_type;
+	gf_flock->l_whence = flock->l_whence;
+	gf_flock->l_start  = flock->l_start;
+	gf_flock->l_len    = flock->l_len;
+	gf_flock->l_pid    = flock->l_pid;
+}
+
+
+int
+glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	struct gf_flock  gf_flock = {0, };
+	struct gf_flock  saved_flock = {0, };
+	fd_t            *fd = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	subvol = glfs_active_subvol (glfd->fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (glfd->fs, subvol, glfd);
+	if (!fd) {
+		ret = -1;
+		errno = EBADFD;
+		goto out;
+	}
+
+	gf_flock_from_flock (&gf_flock, flock);
+	gf_flock_from_flock (&saved_flock, flock);
+	ret = syncop_lk (subvol, fd, cmd, &gf_flock);
+	gf_flock_to_flock (&gf_flock, flock);
+
+	if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW))
+		fd_lk_insert_and_merge (fd, cmd, &saved_flock);
+out:
+	if (fd)
+		fd_unref (fd);
+
+	glfs_subvol_done (glfd->fs, subvol);
+
+	return ret;
+}
+
+
+struct glfs_fd *
+glfs_dup (struct glfs_fd *glfd)
+{
+	xlator_t  *subvol = NULL;
+	fd_t      *fd = NULL;
+	glfs_fd_t *dupfd = NULL;
+	struct glfs *fs = NULL;
+
+	__glfs_entry_fd (glfd);
+
+	fs = glfd->fs;
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	fd = glfs_resolve_fd (fs, subvol, glfd);
+	if (!fd) {
+		errno = EBADFD;
+		goto out;
+	}
+
+	dupfd = glfs_fd_new (fs);
+	if (!dupfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	dupfd->fd = fd_ref (fd);
+out:
+	if (fd)
+		fd_unref (fd);
+	if (dupfd)
+		glfs_fd_bind (dupfd);
+
+	glfs_subvol_done (fs, subvol);
+
+	return dupfd;
+}
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
new file mode 100644
index 000000000..9c707a619
--- /dev/null
+++ b/api/src/glfs-handleops.c
@@ -0,0 +1,1278 @@
+/*
+ *  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+
+#include "glfs-internal.h"
+#include "glfs-mem-types.h"
+#include "syncop.h"
+#include "glfs.h"
+#include "glfs-handles.h"
+
+static void
+glfs_iatt_from_stat (struct stat *stat, int valid, struct iatt *iatt,
+		     int *glvalid)
+{
+	/* validate in args */
+	if ((stat == NULL) || (iatt == NULL) || (glvalid == NULL)) {
+		errno = EINVAL;
+		return;
+	}
+
+	*glvalid = 0;
+
+	if (valid & GFAPI_SET_ATTR_MODE) {
+		iatt->ia_prot = ia_prot_from_st_mode (stat->st_mode);
+		*glvalid |= GF_SET_ATTR_MODE;
+	}
+
+	if (valid & GFAPI_SET_ATTR_UID) {
+		iatt->ia_uid = stat->st_uid;
+		*glvalid |= GF_SET_ATTR_UID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_GID) {
+		iatt->ia_gid = stat->st_gid;
+		*glvalid |= GF_SET_ATTR_GID;
+	}
+
+	if (valid & GFAPI_SET_ATTR_ATIME) {
+		iatt->ia_atime = stat->st_atime;
+		iatt->ia_atime_nsec = ST_ATIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_ATIME;
+	}
+
+	if (valid & GFAPI_SET_ATTR_MTIME) {
+		iatt->ia_mtime = stat->st_mtime;
+		iatt->ia_mtime_nsec = ST_MTIM_NSEC (stat);
+		*glvalid |= GF_SET_ATTR_MTIME;
+	}
+
+	return;
+}
+
+struct glfs_object *
+glfs_h_lookupat (struct glfs *fs, struct glfs_object *parent,
+		 const char *path, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+	struct glfs_object      *object = NULL;
+	loc_t                    loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	if (parent) {
+		inode = glfs_resolve_inode (fs, subvol, parent);
+		if (!inode) {
+			errno = ESTALE;
+			goto out;
+		}
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, &iatt,
+			       0 /*TODO: links? */, 0);
+
+	/* populate out args */
+	if (!ret) {
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_stat (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_stat (subvol, &loc, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_getattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat)
+{
+	int                      ret = 0;
+	xlator_t                *subvol = NULL;
+	inode_t                 *inode = NULL;
+	struct iatt              iatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = glfs_resolve_base (fs, subvol, inode, &iatt);
+
+	/* populate out args */
+	if (!ret && stat) {
+		glfs_iatt_to_stat (fs, &iatt, stat);
+	}
+
+out:
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_setattrs (struct glfs *fs, struct glfs_object *object, struct stat *stat,
+		 int valid)
+{
+	int              ret = -1;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+	struct iatt      iatt = {0, };
+	int              glvalid = 0;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (stat == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* map valid masks from in args */
+	glfs_iatt_from_stat (stat, valid, &iatt, &glvalid);
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_setattr (subvol, &loc, &iatt, glvalid, 0, 0);
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* check types to open */
+	if (IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	if (!IA_ISREG (inode->ia_type)) {
+		ret = -1;
+		errno = EINVAL;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd) {
+		errno = ENOMEM;
+		goto out;
+	}
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* populate loc */
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_open (subvol, &loc, flags, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		glfd->fd->flags = flags;
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+struct glfs_object *
+glfs_h_creat (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      int flags, mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	struct glfs_fd     *glfd = NULL;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	glfd->fd = fd_create (loc.inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	/* fop/op */
+	ret = syncop_create (subvol, &loc, flags, mode, glfd->fd,
+			     xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		   exists), then the glfs_loc_link will not update the
+		   loc.inode, as a result we will have a 0000 GFID that we
+		   would copy out to the object, this needs to be fixed.
+		*/
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	if (glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mkdir (subvol, &loc, mode, xattr_req, &iatt);
+
+	/* populate out args */
+	if ( ret == 0 )  {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+struct glfs_object *
+glfs_h_mknod (struct glfs *fs, struct glfs_object *parent, const char *path,
+	      mode_t mode, dev_t dev, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, path);
+
+	/* fop/op */
+	ret = syncop_mknod (subvol, &loc, mode, dev, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_unlink (struct glfs *fs, struct glfs_object *parent, const char *path)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (path == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, inode, path, &loc, NULL, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	if (!IA_ISDIR(loc.inode->ia_type)) {
+		ret = syncop_unlink (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	} else {
+		ret = syncop_rmdir (subvol, &loc);
+		if (ret != 0) {
+			goto out;
+		}
+	}
+
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_fd *
+glfs_h_opendir (struct glfs *fs, struct glfs_object *object)
+{
+	int              ret = -1;
+	struct glfs_fd  *glfd = NULL;
+	xlator_t        *subvol = NULL;
+	inode_t         *inode = NULL;
+	loc_t            loc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (!IA_ISDIR (inode->ia_type)) {
+		ret = -1;
+		errno = ENOTDIR;
+		goto out;
+	}
+
+	glfd = glfs_fd_new (fs);
+	if (!glfd)
+		goto out;
+
+	INIT_LIST_HEAD (&glfd->entries);
+
+	glfd->fd = fd_create (inode, getpid());
+	if (!glfd->fd) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_opendir (subvol, &loc, glfd->fd);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (ret && glfd) {
+		glfs_fd_destroy (glfd);
+		glfd = NULL;
+	} else {
+		fd_bind (glfd->fd);
+		glfs_fd_bind (glfd);
+	}
+
+	glfs_subvol_done (fs, subvol);
+
+	return glfd;
+}
+
+ssize_t
+glfs_h_extract_handle (struct glfs_object *object, unsigned char *handle,
+		       int len)
+{
+	ssize_t ret = -1;
+
+	/* validate in args */
+	if (object == NULL) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	if (!handle || !len) {
+		ret = GFAPI_HANDLE_LENGTH;
+		goto out;
+	}
+
+	if (len < GFAPI_HANDLE_LENGTH)
+	{
+		errno = ERANGE;
+		goto out;
+	}
+
+	memcpy (handle, object->gfid, GFAPI_HANDLE_LENGTH);
+
+	ret = GFAPI_HANDLE_LENGTH;
+
+out:
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_create_from_handle (struct glfs *fs, unsigned char *handle, int len,
+			   struct stat *stat)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	struct iatt         iatt = {0, };
+	inode_t            *newinode = NULL;
+	xlator_t           *subvol = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (handle == NULL) || (len != GFAPI_HANDLE_LENGTH)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		errno = EIO;
+		goto out;
+	}
+
+	memcpy (loc.gfid, handle, GFAPI_HANDLE_LENGTH);
+
+	newinode = inode_find (subvol->itable, loc.gfid);
+	if (newinode)
+		loc.inode = newinode;
+	else {
+		loc.inode = inode_new (subvol->itable);
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+	}
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		goto out;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+	else {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode linking of %s failed: %s",
+			uuid_utoa (loc.gfid), strerror (errno));
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* populate stat */
+	if (stat)
+		glfs_iatt_to_stat (fs, &iatt, stat);
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		ret = -1;
+		goto out;
+	}
+
+	/* populate the return object */
+	object->inode = newinode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+out:
+	/* TODO: Check where the inode ref is being held? */
+	loc_wipe (&loc);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_close (struct glfs_object *object)
+{
+	/* Release the held reference */
+	inode_unref (object->inode);
+	GF_FREE (object);
+
+	return 0;
+}
+
+int
+glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_truncate (subvol, &loc, (off_t)offset);
+
+	/* populate out args */
+	if (ret == 0)
+		ret = glfs_loc_unlink (&loc);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+struct glfs_object *
+glfs_h_symlink (struct glfs *fs, struct glfs_object *parent, const char *name,
+		const char *data, struct stat *stat)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	loc_t               loc = {0, };
+	struct iatt         iatt = {0, };
+	uuid_t              gfid;
+	dict_t             *xattr_req = NULL;
+	struct glfs_object *object = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (parent == NULL) || (name == NULL) ||
+		(data == NULL)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, parent);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	xattr_req = dict_new ();
+	if (!xattr_req) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	uuid_generate (gfid);
+	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+	if (ret) {
+		ret = -1;
+		errno = ENOMEM;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_PINODE (inode, loc, ret, errno, out, name);
+
+	/* fop/op */
+	ret = syncop_symlink (subvol, &loc, data, xattr_req, &iatt);
+
+	/* populate out args */
+	if (ret == 0) {
+		/* TODO: If the inode existed in the cache (say file already
+		 * exists), then the glfs_loc_link will not update the
+		 * loc.inode, as a result we will have a 0000 GFID that we
+		 * would copy out to the object, this needs to be fixed.
+		 */
+		ret = glfs_loc_link (&loc, &iatt);
+		if (ret != 0) {
+			goto out;
+		}
+
+		if (stat)
+			glfs_iatt_to_stat (fs, &iatt, stat);
+
+		ret = glfs_create_object (&loc, &object);
+	}
+
+out:
+	if (ret && object != NULL) {
+		glfs_h_close (object);
+		object = NULL;
+	}
+
+	loc_wipe(&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (xattr_req)
+		dict_unref (xattr_req);
+
+	glfs_subvol_done (fs, subvol);
+
+	return object;
+}
+
+int
+glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		 size_t bufsiz)
+{
+	loc_t               loc = {0, };
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	char               *linkval = NULL;
+
+	/* validate in args */
+	if ((fs == NULL) || (object == NULL) || (buf == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, object);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, loc, out);
+
+	/* fop/op */
+	ret = syncop_readlink (subvol, &loc, &linkval, bufsiz);
+
+	/* populate out args */
+	if (ret > 0)
+		memcpy (buf, linkval, ret);
+
+out:
+	loc_wipe (&loc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (linkval)
+		GF_FREE (linkval);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_link (struct glfs *fs, struct glfs_object *linksrc,
+	     struct glfs_object *parent, const char *name)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *inode = NULL;
+	inode_t            *pinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (linksrc == NULL) || (parent == NULL) ||
+		(name == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if (!subvol) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	inode = glfs_resolve_inode (fs, subvol, linksrc);
+	if (!inode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	if (inode->ia_type == IA_IFDIR) {
+		ret = -1;
+		errno = EISDIR;
+		goto out;
+	}
+
+	GLFS_LOC_FILL_INODE (inode, oldloc, out);
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	pinode = glfs_resolve_inode (fs, subvol, parent);
+	if (!pinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	/* setup newloc based on parent */
+	newloc.parent = inode_ref (pinode);
+	newloc.name = name;
+	ret = glfs_loc_touchup (&newloc);
+	if (ret != 0) {
+		errno = EINVAL;
+		goto out;
+	}
+
+	/* Filling the inode of the hard link to be same as that of the
+	 * original file
+	 */
+	newloc.inode = inode_ref (inode);
+
+	/* fop/op */
+	ret = syncop_link (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		/* TODO: No iatt to pass as there has been no lookup */
+		ret = glfs_loc_link (&newloc, NULL);
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (inode)
+		inode_unref (inode);
+
+	if (pinode)
+		inode_unref (pinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
+
+int
+glfs_h_rename (struct glfs *fs, struct glfs_object *olddir, const char *oldname,
+	       struct glfs_object *newdir, const char *newname)
+{
+	int                 ret = -1;
+	xlator_t           *subvol = NULL;
+	inode_t            *oldpinode = NULL;
+	inode_t            *newpinode = NULL;
+	loc_t               oldloc = {0, };
+	loc_t               newloc = {0, };
+	struct iatt         oldiatt = {0, };
+	struct iatt         newiatt = {0, };
+
+	/* validate in args */
+	if ((fs == NULL) || (olddir == NULL) || (oldname == NULL) ||
+		(newdir == NULL) || (newname == NULL)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	__glfs_entry_fs (fs);
+
+	/* get the active volume */
+	subvol = glfs_active_subvol (fs);
+	if ( !subvol ) {
+		ret = -1;
+		errno = EIO;
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	oldpinode = glfs_resolve_inode (fs, subvol, olddir);
+	if (!oldpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, oldpinode, oldname, &oldloc,
+			       &oldiatt, 0 , 0);
+	if (ret != 0) {
+		goto out;
+	}
+
+	/* get/refresh the in arg objects inode in correlation to the xlator */
+	newpinode = glfs_resolve_inode (fs, subvol, newdir);
+	if (!newpinode) {
+		errno = ESTALE;
+		goto out;
+	}
+
+	ret = glfs_resolve_at (fs, subvol, newpinode, newname, &newloc,
+			       &newiatt, 0, 0);
+
+	if (ret && errno != ENOENT && newloc.parent)
+		goto out;
+
+	if (newiatt.ia_type != IA_INVAL) {
+		if ((oldiatt.ia_type == IA_IFDIR) !=
+			(newiatt.ia_type == IA_IFDIR)) {
+			/* Either both old and new must be dirs,
+			 * or both must be non-dirs. Else, fail.
+			 */
+			ret = -1;
+			errno = EISDIR;
+			goto out;
+		}
+	}
+
+	/* TODO: check if new or old is a prefix of the other, and fail EINVAL */
+
+	ret = syncop_rename (subvol, &oldloc, &newloc);
+
+	if (ret == 0)
+		inode_rename (oldloc.parent->table, oldloc.parent, oldloc.name,
+			      newloc.parent, newloc.name, oldloc.inode,
+			      &oldiatt);
+
+out:
+	loc_wipe (&oldloc);
+	loc_wipe (&newloc);
+
+	if (oldpinode)
+		inode_unref (oldpinode);
+
+	if (newpinode)
+		inode_unref (newpinode);
+
+	glfs_subvol_done (fs, subvol);
+
+	return ret;
+}
diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h
new file mode 100644
index 000000000..437f2cbc8
--- /dev/null
+++ b/api/src/glfs-handles.h
@@ -0,0 +1,143 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLFS_HANDLES_H
+#define _GLFS_HANDLES_H
+
+#include "glfs.h"
+
+/* GLFS OBJECT BASED OPERATIONS
+ *
+ * The following APIs are introduced to provide an API framework that can work
+ * with gluster objects (files and directories), instead of absolute paths.
+ *
+ * The following API set can be related to the POSIX *at interfaces (like
+ * openat (2)). The intention of these APIs is to be able to operate based
+ * on parent object and looking up or creating child objects within, OR to be
+ * used on the actual object thus looked up or created, and retrieve information
+ * regarding the same.
+ *
+ * The APIs also provide for generating an opaque invariant handle to the
+ * object, that can later be used to lookup the object, instead of the regular
+ * glfs_h_* variants. The APIs that provide this behaviour are,
+ * glfs_h_extract_handle and glfs_h_create_from_handle.
+ *
+ * The object handles can be transitioned to fd based operations as supported
+ * by glfs.h calls, using the glfs_h_open call. This provides a way to move
+ * from objects to fd's akin to moving from path to fd for required operations.
+ *
+ * NOTE: The opaque invariant handle is the GFID of the object in reality, but
+ * maintained as an opaque data value, for potential internal changes to the
+ * same without impacting the caller.
+ *
+ * NOTE: Currently looking up an object can create multiple object handles to
+ * the same, i.e distinct glfs_object *. Hence each such looked up or received
+ * handle from other calls, would need to be closed. In the future, for a given
+ * object these pointers would be the same, and an ease of use API to forget all
+ * instances of this bject would be provided (instead of a per lookup close).
+ * This should not change the APIs in their current form.
+ *
+ */
+
+/* Values for valid falgs to be used when using XXXsetattr, to set multiple
+ attribute values passed via the related stat structure.
+ */
+#define GFAPI_SET_ATTR_MODE  0x1
+#define GFAPI_SET_ATTR_UID   0x2
+#define GFAPI_SET_ATTR_GID   0x4
+#define GFAPI_SET_ATTR_SIZE  0x8
+#define GFAPI_SET_ATTR_ATIME 0x10
+#define GFAPI_SET_ATTR_MTIME 0x20
+
+/* Handle length for object handles returned from glfs_h_extract_handle or
+ * glfs_h_create_from_handle */
+#define GFAPI_HANDLE_LENGTH 16
+
+__BEGIN_DECLS
+
+/*
+ * Notes:
+ *
+ * The file object handle. One per looked up, created file/directory
+ *
+ * This had been introduced to facilitate gfid/inode based gfapi
+ * - a requirement introduced by nfs-ganesha
+ */
+struct glfs_object;
+typedef struct glfs_object glfs_object_t;
+
+/* Handle based operations */
+/* Operations that generate handles */
+struct glfs_object *glfs_h_lookupat (struct glfs *fs,
+				     struct glfs_object *parent,
+				     const char *path, struct stat *stat);
+
+struct glfs_object *glfs_h_creat (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, int flags, mode_t mode,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mkdir (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t flags,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_mknod (struct glfs *fs, struct glfs_object *parent,
+				  const char *path, mode_t mode, dev_t dev,
+				  struct stat *sb);
+
+struct glfs_object *glfs_h_symlink (struct glfs *fs, struct glfs_object *parent,
+				    const char *name, const char *data,
+				    struct stat *stat);
+
+/* Operations on the actual objects */
+int glfs_h_unlink (struct glfs *fs, struct glfs_object *parent,
+		   const char *path);
+
+int glfs_h_close (struct glfs_object *object);
+
+int glfs_caller_specific_init (void *uid_caller_key, void *gid_caller_key,
+			       void *future);
+
+int glfs_h_truncate (struct glfs *fs, struct glfs_object *object, off_t offset);
+
+int glfs_h_stat(struct glfs *fs, struct glfs_object *object, struct stat *stat);
+
+int glfs_h_getattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *stat);
+
+int glfs_h_setattrs (struct glfs *fs, struct glfs_object *object,
+		     struct stat *sb, int valid);
+
+int glfs_h_readlink (struct glfs *fs, struct glfs_object *object, char *buf,
+		     size_t bufsiz);
+
+int glfs_h_link (struct glfs *fs, struct glfs_object *linktgt,
+		 struct glfs_object *parent, const char *name);
+
+int glfs_h_rename (struct glfs *fs, struct glfs_object *olddir,
+		   const char *oldname, struct glfs_object *newdir,
+		   const char *newname);
+
+/* Operations enabling opaque invariant handle to object transitions */
+ssize_t glfs_h_extract_handle (struct glfs_object *object,
+			       unsigned char *handle, int len);
+
+struct glfs_object *glfs_h_create_from_handle (struct glfs *fs,
+					       unsigned char *handle, int len,
+					       struct stat *stat);
+
+/* Operations enabling object handles to fd transitions */
+struct glfs_fd *glfs_h_opendir (struct glfs *fs, struct glfs_object *object);
+
+struct glfs_fd *glfs_h_open (struct glfs *fs, struct glfs_object *object,
+			     int flags);
+
+__END_DECLS
+
+#endif /* !_GLFS_HANDLES_H */
\ No newline at end of file
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index 6380cd2ae..ec1d5579d 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -14,6 +14,46 @@
 
 #include "xlator.h"
 
+#define GLFS_SYMLINK_MAX_FOLLOW 2048
+
+#define DEFAULT_REVAL_COUNT 1
+
+#define ESTALE_RETRY(ret,errno,reval,loc,label) do {	\
+	if (ret == -1 && errno == ESTALE) {	        \
+		if (reval < DEFAULT_REVAL_COUNT) {	\
+			reval++;			\
+			loc_wipe (loc);			\
+			goto label;			\
+		}					\
+	}						\
+	} while (0)
+
+#define GLFS_LOC_FILL_INODE(oinode, loc, label) do {   \
+	loc.inode = inode_ref (oinode);                \
+	uuid_copy (loc.gfid, oinode->gfid);            \
+	ret = glfs_loc_touchup (&loc);                 \
+	if (ret != 0) {                                \
+		errno = EINVAL;                        \
+		goto label;                            \
+	}                                              \
+	} while (0)
+
+#define GLFS_LOC_FILL_PINODE(pinode, loc, ret, errno, label, path) do {   \
+	loc.inode = inode_new (pinode->table);                            \
+	if (!loc.inode) {                                                 \
+		ret = -1;                                                 \
+		errno = ENOMEM;                                           \
+		goto label;                                               \
+	}                                                                 \
+	loc.parent = inode_ref (pinode);                                  \
+	loc.name = path;                                                  \
+	ret = glfs_loc_touchup (&loc);                                    \
+	if (ret != 0) {                                                   \
+		errno = EINVAL;                                           \
+		goto label;                                               \
+	}                                                                 \
+	} while (0)
+
 struct glfs;
 
 typedef int (*glfs_init_cbk) (struct glfs *fs, int ret);
@@ -33,14 +73,37 @@ struct glfs {
 	int                 err;
 
 	xlator_t           *active_subvol;
+	xlator_t           *next_subvol;
+	xlator_t           *old_subvol;
 
 	char               *oldvolfile;
 	ssize_t             oldvollen;
+
+	inode_t            *cwd;
+
+	uint32_t            dev_id; /* Used to fill st_dev in struct stat */
+
+	struct list_head    openfds;
+
+	gf_boolean_t        migration_in_progress;
 };
 
 struct glfs_fd {
-	off_t     offset;
-	fd_t     *fd;
+	struct list_head   openfds;
+	struct glfs       *fs;
+	off_t              offset;
+	fd_t              *fd; /* Currently guared by @fs->mutex. TODO: per-glfd lock */
+	struct list_head   entries;
+	gf_dirent_t       *next;
+	struct dirent     *readdirbuf;
+};
+
+/* glfs object handle introduced for the alternate gfapi implementation based
+   on glfs handles/gfid/inode
+*/
+struct glfs_object {
+        inode_t         *inode;
+        uuid_t          gfid;
 };
 
 #define DEFAULT_EVENT_POOL_SIZE           16384
@@ -52,8 +115,14 @@ int glfs_mgmt_init (struct glfs *fs);
 void glfs_init_done (struct glfs *fs, int ret);
 int glfs_process_volfp (struct glfs *fs, FILE *fp);
 int glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
-		  struct iatt *iatt);
-void glfs_first_lookup (xlator_t *subvol);
+		  struct iatt *iatt, int reval);
+int glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *path, loc_t *loc,
+		   struct iatt *iatt, int reval);
+fd_t *glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+fd_t *__glfs_migrate_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd);
+
+int glfs_first_lookup (xlator_t *subvol);
 
 static inline void
 __glfs_entry_fs (struct glfs *fs)
@@ -69,10 +138,63 @@ __glfs_entry_fd (struct glfs_fd *fd)
 }
 
 
+/*
+  By default all lock attempts from user context must
+  use glfs_lock() and glfs_unlock(). This allows
+  for a safe implementation of graph migration where
+  we can give up the mutex during syncop calls so
+  that bottom up calls (particularly CHILD_UP notify)
+  can do a mutex_lock() on @glfs without deadlocking
+  the filesystem
+*/
+static inline int
+glfs_lock (struct glfs *fs)
+{
+	pthread_mutex_lock (&fs->mutex);
+
+	while (!fs->init)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	while (fs->migration_in_progress)
+		pthread_cond_wait (&fs->cond, &fs->mutex);
+
+	return 0;
+}
+
+
+static inline void
+glfs_unlock (struct glfs *fs)
+{
+	pthread_mutex_unlock (&fs->mutex);
+}
+
+
 void glfs_fd_destroy (struct glfs_fd *glfd);
 
-xlator_t * glfs_fd_subvol (struct glfs_fd *glfd);
+struct glfs_fd *glfs_fd_new (struct glfs *fs);
+void glfs_fd_bind (struct glfs_fd *glfd);
 
 xlator_t * glfs_active_subvol (struct glfs *fs);
+xlator_t * __glfs_active_subvol (struct glfs *fs);
+void glfs_subvol_done (struct glfs *fs, xlator_t *subvol);
+
+inode_t * glfs_refresh_inode (xlator_t *subvol, inode_t *inode);
+
+inode_t *glfs_cwd_get (struct glfs *fs);
+int glfs_cwd_set (struct glfs *fs, inode_t *inode);
+inode_t *glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+			     struct glfs_object *object);
+int glfs_create_object (loc_t *loc, struct glfs_object **retobject);
+int __glfs_cwd_set (struct glfs *fs, inode_t *inode);
+
+int glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		       struct iatt *iatt);
+int glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+                     const char *origpath, loc_t *loc, struct iatt *iatt,
+                     int follow, int reval);
+int glfs_loc_touchup (loc_t *loc);
+void glfs_iatt_to_stat (struct glfs *fs, struct iatt *iatt, struct stat *stat);
+int glfs_loc_link (loc_t *loc, struct iatt *iatt);
+int glfs_loc_unlink (loc_t *loc);
 
 #endif /* !_GLFS_INTERNAL_H */
diff --git a/api/src/glfs-master.c b/api/src/glfs-master.c
index 0806c3077..c02534c18 100644
--- a/api/src/glfs-master.c
+++ b/api/src/glfs-master.c
@@ -22,26 +22,58 @@
 
 #include "xlator.h"
 #include "glusterfs.h"
+
 #include "glfs-internal.h"
+#include "glfs-mem-types.h"
 
 
 int
 glfs_graph_setup (struct glfs *fs, glusterfs_graph_t *graph)
 {
-	if (fs->active_subvol == graph->top)
-		return 0;
+	xlator_t      *new_subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+	inode_table_t *itable = NULL;
+	int            ret = -1;
+
+	new_subvol = graph->top;
 
+	/* This is called in a bottom-up context, it should specifically
+	   NOT be glfs_lock()
+	*/
 	pthread_mutex_lock (&fs->mutex);
 	{
-		fs->active_subvol = graph->top;
-		pthread_cond_broadcast (&fs->cond);
+		if (new_subvol->switched ||
+		    new_subvol == fs->active_subvol ||
+		    new_subvol == fs->next_subvol) {
+			/* Spurious CHILD_UP event on old graph */
+			ret = 0;
+			goto unlock;
+		}
+
+		if (!new_subvol->itable) {
+			itable = inode_table_new (131072, new_subvol);
+			if (!itable) {
+				errno = ENOMEM;
+				ret = -1;
+				goto unlock;
+			}
+
+			new_subvol->itable = itable;
+		}
+
+		old_subvol = fs->next_subvol;
+		fs->next_subvol = new_subvol;
+		fs->next_subvol->winds++; /* first ref */
+		ret = 0;
 	}
+unlock:
 	pthread_mutex_unlock (&fs->mutex);
 
-	gf_log ("glfs-master", GF_LOG_INFO, "switched to graph %s (%d)",
-		uuid_utoa ((unsigned char *)graph->graph_uuid), graph->id);
+	if (old_subvol)
+		/* wasn't picked up so far, skip */
+		glfs_subvol_done (fs, old_subvol);
 
-	return 0;
+	return ret;
 }
 
 
@@ -83,6 +115,18 @@ notify (xlator_t *this, int event, void *data, ...)
 int
 mem_acct_init (xlator_t *this)
 {
+	int ret = -1;
+
+	if (!this)
+		return ret;
+
+	ret = xlator_mem_acct_init (this, glfs_mt_end + 1);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to initialise "
+                        "memory accounting");
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -101,13 +145,10 @@ fini (xlator_t *this)
 }
 
 
-struct xlator_dumpops dumpops = {
-};
+struct xlator_dumpops dumpops;
 
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
diff --git a/api/src/glfs-mem-types.h b/api/src/glfs-mem-types.h
index e2e9b4c1e..3301b3da5 100644
--- a/api/src/glfs-mem-types.h
+++ b/api/src/glfs-mem-types.h
@@ -16,13 +16,16 @@
 #define GF_MEM_TYPE_START (gf_common_mt_end + 1)
 
 enum glfs_mem_types_ {
-        glfs_mt_glfs_t,
+        glfs_mt_glfs_t = GF_MEM_TYPE_START,
         glfs_mt_call_pool_t,
         glfs_mt_xlator_t,
 	glfs_mt_glfs_fd_t,
 	glfs_mt_glfs_io_t,
 	glfs_mt_volfile_t,
-        glfs_mt_end
+	glfs_mt_xlator_cmdline_option_t,
+	glfs_mt_glfs_object_t,
+	glfs_mt_readdirbuf_t,
+	glfs_mt_end
 
 };
 #endif
diff --git a/api/src/glfs-mgmt.c b/api/src/glfs-mgmt.c
index 2ead93863..6843e9cb3 100644
--- a/api/src/glfs-mgmt.c
+++ b/api/src/glfs-mgmt.c
@@ -114,7 +114,7 @@ mgmt_cbk_event (struct rpc_clnt *rpc, void *mydata, void *data)
 }
 
 
-rpcclnt_cb_actor_t gluster_cbk_actors[] = {
+rpcclnt_cb_actor_t mgmt_cbk_actors[] = {
 	[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
 	[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
 				 mgmt_cbk_event},
@@ -125,7 +125,7 @@ struct rpcclnt_cb_program mgmt_cbk_prog = {
 	.progname  = "GlusterFS Callback",
 	.prognum   = GLUSTER_CBK_PROGRAM,
 	.progver   = GLUSTER_CBK_VERSION,
-	.actors	   = gluster_cbk_actors,
+	.actors	   = mgmt_cbk_actors,
 	.numactors = GF_CBK_MAXVALUE,
 };
 
@@ -203,162 +203,6 @@ out:
 
 
 static int
-xlator_equal_rec (xlator_t *xl1, xlator_t *xl2)
-{
-	xlator_list_t *trav1 = NULL;
-	xlator_list_t *trav2 = NULL;
-	int	       ret   = 0;
-
-	if (xl1 == NULL || xl2 == NULL) {
-		gf_log ("xlator", GF_LOG_DEBUG, "invalid argument");
-		return -1;
-	}
-
-	trav1 = xl1->children;
-	trav2 = xl2->children;
-
-	while (trav1 && trav2) {
-		ret = xlator_equal_rec (trav1->xlator, trav2->xlator);
-		if (ret) {
-			gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-				"xlators children not equal");
-			goto out;
-		}
-
-		trav1 = trav1->next;
-		trav2 = trav2->next;
-	}
-
-	if (trav1 || trav2) {
-		ret = -1;
-		goto out;
-	}
-
-	if (strcmp (xl1->name, xl2->name)) {
-		ret = -1;
-		goto out;
-	}
-out :
-	return ret;
-}
-
-
-static gf_boolean_t
-is_graph_topology_equal (glusterfs_graph_t *graph1,
-			 glusterfs_graph_t *graph2)
-{
-	xlator_t    *trav1    = NULL;
-	xlator_t    *trav2    = NULL;
-	gf_boolean_t ret      = _gf_true;
-
-	trav1 = graph1->first;
-	trav2 = graph2->first;
-
-	ret = xlator_equal_rec (trav1, trav2);
-
-	if (ret) {
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"graphs are not equal");
-		ret = _gf_false;
-		goto out;
-	}
-
-	ret = _gf_true;
-	gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-		"graphs are equal");
-
-out:
-	return ret;
-}
-
-
-/* Function has 3types of return value 0, -ve , 1
- *   return 0	       =======> reconfiguration of options has succeeded
- *   return 1	       =======> the graph has to be reconstructed and all the xlators should be inited
- *   return -1(or -ve) =======> Some Internal Error occurred during the operation
- */
-static int
-glusterfs_volfile_reconfigure (struct glfs *fs, FILE *newvolfile_fp)
-{
-	glusterfs_graph_t *oldvolfile_graph = NULL;
-	glusterfs_graph_t *newvolfile_graph = NULL;
-	FILE		  *oldvolfile_fp    = NULL;
-	glusterfs_ctx_t	  *ctx		    = NULL;
-
-	int ret = -1;
-
-	oldvolfile_fp = tmpfile ();
-	if (!oldvolfile_fp)
-		goto out;
-
-	if (!fs->oldvollen) {
-		ret = 1; // Has to call INIT for the whole graph
-		goto out;
-	}
-	fwrite (fs->oldvolfile, fs->oldvollen, 1, oldvolfile_fp);
-	fflush (oldvolfile_fp);
-	if (ferror (oldvolfile_fp)) {
-		goto out;
-	}
-
-	oldvolfile_graph = glusterfs_graph_construct (oldvolfile_fp);
-	if (!oldvolfile_graph) {
-		goto out;
-	}
-
-	newvolfile_graph = glusterfs_graph_construct (newvolfile_fp);
-	if (!newvolfile_graph) {
-		goto out;
-	}
-
-	if (!is_graph_topology_equal (oldvolfile_graph,
-				      newvolfile_graph)) {
-
-		ret = 1;
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"Graph topology not equal(should call INIT)");
-		goto out;
-	}
-
-	gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-		"Only options have changed in the new "
-		"graph");
-
-	ctx = fs->ctx;
-
-	if (!ctx) {
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"glusterfs_ctx_get() returned NULL");
-		goto out;
-	}
-
-	oldvolfile_graph = ctx->active;
-
-	if (!oldvolfile_graph) {
-		gf_log ("glfs-mgmt", GF_LOG_ERROR,
-			"glusterfs_ctx->active is NULL");
-		goto out;
-	}
-
-	/* */
-	ret = glusterfs_graph_reconfigure (oldvolfile_graph,
-					   newvolfile_graph);
-	if (ret) {
-		gf_log ("glfs-mgmt", GF_LOG_DEBUG,
-			"Could not reconfigure new options in old graph");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	if (oldvolfile_fp)
-		fclose (oldvolfile_fp);
-
-	return ret;
-}
-
-
-static int
 glusterfs_oldvolfile_update (struct glfs *fs, char *volfile, ssize_t size)
 {
 	int ret = -1;
@@ -416,6 +260,7 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
 		gf_log (frame->this->name, GF_LOG_ERROR,
 			"failed to get the 'volume file' from server");
 		ret = -1;
+                errno = rsp.op_errno;
 		goto out;
 	}
 
@@ -450,7 +295,8 @@ mgmt_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
 	*  return -1(or -ve) =======> Some Internal Error occurred during the operation
 	*/
 
-	ret = glusterfs_volfile_reconfigure (fs, tmpfp);
+	ret = glusterfs_volfile_reconfigure (fs->oldvollen, tmpfp, fs->ctx,
+                                             fs->oldvolfile);
 	if (ret == 0) {
 		gf_log ("glusterfsd-mgmt", GF_LOG_DEBUG,
 			"No need to re-load volfile, reconfigure done");
@@ -477,6 +323,14 @@ out:
 	if (rsp.spec)
 		free (rsp.spec);
 
+        // Stop if server is running at an unsupported op-version
+        if (ENOTSUP == ret) {
+                gf_log ("mgmt", GF_LOG_ERROR, "Server is operating at an "
+                        "op-version which is not supported");
+                errno = ENOTSUP;
+                glfs_init_done (fs, -1);
+        }
+
 	if (ret && ctx && !ctx->active) {
 		/* Do it only for the first time */
 		/* Failed to get the volume file, something wrong,
@@ -484,8 +338,11 @@ out:
 		gf_log ("glfs-mgmt", GF_LOG_ERROR,
 			"failed to fetch volume file (key:%s)",
 			ctx->cmd_args.volfile_id);
-		if (!need_retry)
+		if (!need_retry) {
+                        if (!errno)
+                                errno = EINVAL;
 			glfs_init_done (fs, -1);
+                }
 	}
 
 	if (tmpfp)
@@ -503,6 +360,7 @@ glfs_volfile_fetch (struct glfs *fs)
 	int		  ret = 0;
 	call_frame_t	 *frame = NULL;
 	glusterfs_ctx_t	 *ctx = NULL;
+        dict_t           *dict = NULL;
 
 	ctx = fs->ctx;
 	cmd_args = &ctx->cmd_args;
@@ -512,10 +370,41 @@ glfs_volfile_fetch (struct glfs *fs)
 	req.key = cmd_args->volfile_id;
 	req.flags = 0;
 
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        // Set the supported min and max op-versions, so glusterd can make a
+        // decision
+        ret = dict_set_int32 (dict, "min-op-version", GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set min-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "max-op-version", GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set max-op-version"
+                        " in request dict");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &req.xdata.xdata_val,
+                                           &req.xdata.xdata_len);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to serialize dictionary");
+                goto out;
+        }
+
 	ret = mgmt_submit_request (&req, frame, ctx, &clnt_handshake_prog,
 				   GF_HNDSK_GETSPEC, mgmt_getspec_cbk,
 				   (xdrproc_t)xdr_gf_getspec_req);
-	return ret;
+out:
+        return ret;
 }
 
 
@@ -544,9 +433,10 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
 			gf_log ("glfs-mgmt", GF_LOG_INFO,
 				"%d connect attempts left",
 				cmd_args->max_connect_attempts);
-			if (0 >= cmd_args->max_connect_attempts)
+			if (0 >= cmd_args->max_connect_attempts) {
+                                errno = ENOTCONN;
 				glfs_init_done (fs, -1);
-				break;
+                        }
 		}
 		break;
 	case RPC_CLNT_CONNECT:
@@ -555,10 +445,11 @@ mgmt_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
 		ret = glfs_volfile_fetch (fs);
 		if (ret && ctx && (ctx->active == NULL)) {
 			/* Do it only for the first time */
-			/* Exit the process.. there is some wrong options */
+			/* Exit the process.. there are some wrong options */
 			gf_log ("glfs-mgmt", GF_LOG_ERROR,
 				"failed to fetch volume file (key:%s)",
 				ctx->cmd_args.volfile_id);
+                        errno = EINVAL;
 			glfs_init_done (fs, -1);
 		}
 
diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c
index c03c4154d..4ca2eb6fc 100644
--- a/api/src/glfs-resolve.c
+++ b/api/src/glfs-resolve.c
@@ -32,9 +32,11 @@
 
 #include "glfs-internal.h"
 
+#define graphid_str(subvol) (uuid_utoa((unsigned char *)subvol->graph->graph_uuid))
 
-void
-glfs_first_lookup (xlator_t *subvol)
+
+int
+glfs_first_lookup_safe (xlator_t *subvol)
 {
 	loc_t  loc = {0, };
 	int    ret = -1;
@@ -49,18 +51,100 @@ glfs_first_lookup (xlator_t *subvol)
 
 	gf_log (subvol->name, GF_LOG_DEBUG, "first lookup complete %d", ret);
 
-	return;
+	return ret;
 }
 
 
 int
+__glfs_first_lookup (struct glfs *fs, xlator_t *subvol)
+{
+	int ret = -1;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		ret = glfs_first_lookup_safe (subvol);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return ret;
+}
+
+
+inode_t *
+glfs_refresh_inode_safe (xlator_t *subvol, inode_t *oldinode)
+{
+	loc_t        loc = {0, };
+	int          ret = -1;
+	struct iatt  iatt = {0, };
+	inode_t     *newinode = NULL;
+
+
+	if (!oldinode)
+		return NULL;
+
+	if (oldinode->table->xl == subvol)
+		return inode_ref (oldinode);
+
+	newinode = inode_find (subvol->itable, oldinode->gfid);
+	if (newinode)
+		return newinode;
+
+	uuid_copy (loc.gfid, oldinode->gfid);
+	loc.inode = inode_new (subvol->itable);
+	if (!loc.inode)
+		return NULL;
+
+	ret = syncop_lookup (subvol, &loc, 0, &iatt, 0, 0);
+
+	if (ret) {
+		gf_log (subvol->name, GF_LOG_WARNING,
+			"inode refresh of %s failed: %s",
+			uuid_utoa (oldinode->gfid), strerror (errno));
+		loc_wipe (&loc);
+		return NULL;
+	}
+
+	newinode = inode_link (loc.inode, 0, 0, &iatt);
+	if (newinode)
+		inode_lookup (newinode);
+
+	loc_wipe (&loc);
+
+	return newinode;
+}
+
+
+inode_t *
+__glfs_refresh_inode (struct glfs *fs, xlator_t *subvol, inode_t *inode)
+{
+	inode_t *newinode = NULL;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newinode = glfs_refresh_inode_safe (subvol, inode);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newinode;
+}
+
+int
 glfs_loc_touchup (loc_t *loc)
 {
 	char *path = NULL;
 	int   ret = -1;
 	char *bn = NULL;
 
-	ret = inode_path (loc->parent, loc->name, &path);
+	if (loc->parent)
+		ret = inode_path (loc->parent, loc->name, &path);
+	else
+		ret = inode_path (loc->inode, 0, &path);
 
 	loc->path = path;
 
@@ -80,9 +164,63 @@ out:
 }
 
 
+int
+glfs_resolve_symlink (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		      char **lpath)
+{
+	loc_t  loc = {0, };
+	char  *path = NULL;
+	char  *rpath = NULL;
+	int    ret = -1;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+	ret = inode_path (inode, NULL, &rpath);
+	if (ret < 0)
+		goto out;
+	loc.path = rpath;
+
+	ret = syncop_readlink (subvol, &loc, &path, 4096);
+
+	if (ret < 0)
+		goto out;
+
+	if (lpath)
+		*lpath = path;
+out:
+	loc_wipe (&loc);
+	return ret;
+}
+
+
+int
+glfs_resolve_base (struct glfs *fs, xlator_t *subvol, inode_t *inode,
+		   struct iatt *iatt)
+{
+	loc_t       loc = {0, };
+	int         ret = -1;
+	char       *path = NULL;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	ret = inode_path (loc.inode, NULL, &path);
+	loc.path = path;
+	if (ret < 0)
+		goto out;
+
+	ret = syncop_lookup (subvol, &loc, NULL, iatt, NULL, NULL);
+out:
+	loc_wipe (&loc);
+
+	return ret;
+}
+
+
 inode_t *
 glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
-			const char *component, struct iatt *iatt)
+			const char *component, struct iatt *iatt,
+			int force_lookup)
 {
 	loc_t        loc = {0, };
 	inode_t     *inode = NULL;
@@ -98,23 +236,23 @@ glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
 	loc.parent = inode_ref (parent);
 	uuid_copy (loc.pargfid, parent->gfid);
 
-	xattr_req = dict_new ();
-	if (!xattr_req) {
-		errno = ENOMEM;
-		goto out;
-	}
 
-	ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
-	if (ret) {
-		errno = ENOMEM;
-		goto out;
-	}
-
-	loc.inode = inode_grep (parent->table, parent, component);
+	if (strcmp (component, ".") == 0)
+		loc.inode = inode_ref (parent);
+	else if (strcmp (component, "..") == 0)
+		loc.inode = inode_parent (parent, 0, 0);
+	else
+		loc.inode = inode_grep (parent->table, parent, component);
 
 	if (loc.inode) {
 		uuid_copy (loc.gfid, loc.inode->gfid);
 		reval = 1;
+
+		if (!force_lookup) {
+			inode = inode_ref (loc.inode);
+			ciatt.ia_type = inode->ia_type;
+			goto found;
+		}
 	} else {
 		uuid_generate (gfid);
 		loc.inode = inode_new (parent->table);
@@ -123,20 +261,35 @@ glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
 	if (!loc.inode)
 		goto out;
 
-
 	glret = glfs_loc_touchup (&loc);
 	if (glret < 0) {
 		ret = -1;
 		goto out;
 	}
 
-	ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt, NULL, NULL);
+	ret = syncop_lookup (subvol, &loc, NULL, &ciatt, NULL, NULL);
 	if (ret && reval) {
 		inode_unref (loc.inode);
 		loc.inode = inode_new (parent->table);
-		if (!loc.inode)
+		if (!loc.inode) {
+			errno = ENOMEM;
+			goto out;
+		}
+
+		xattr_req = dict_new ();
+		if (!xattr_req) {
+			errno = ENOMEM;
 			goto out;
+		}
+
 		uuid_generate (gfid);
+
+		ret = dict_set_static_bin (xattr_req, "gfid-req", gfid, 16);
+		if (ret) {
+			errno = ENOMEM;
+			goto out;
+		}
+
 		ret = syncop_lookup (subvol, &loc, xattr_req, &ciatt,
 				     NULL, NULL);
 	}
@@ -144,13 +297,14 @@ glfs_resolve_component (struct glfs *fs, xlator_t *subvol, inode_t *parent,
 		goto out;
 
 	inode = inode_link (loc.inode, loc.parent, component, &ciatt);
+found:
 	if (inode)
 		inode_lookup (inode);
 	if (iatt)
 		*iatt = ciatt;
 out:
 	if (xattr_req)
-		dict_destroy (xattr_req);
+		dict_unref (xattr_req);
 
 	loc_wipe (&loc);
 
@@ -159,8 +313,9 @@ out:
 
 
 int
-glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
-	      loc_t *loc, struct iatt *iatt)
+glfs_resolve_at (struct glfs *fs, xlator_t *subvol, inode_t *at,
+		 const char *origpath, loc_t *loc, struct iatt *iatt,
+		 int follow, int reval)
 {
 	inode_t    *inode = NULL;
 	inode_t    *parent = NULL;
@@ -178,7 +333,17 @@ glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
 	}
 
 	parent = NULL;
-	inode = inode_ref (subvol->itable->root);
+	if (at && path[0] != '/') {
+		/* A relative resolution of a path which starts with '/'
+		   is equal to an absolute path resolution.
+		*/
+		inode = inode_ref (at);
+	} else {
+		inode = inode_ref (subvol->itable->root);
+
+		if (strcmp (path, "/") == 0)
+			glfs_resolve_base (fs, subvol, inode, &ciatt);
+	}
 
 	for (component = strtok_r (path, "/", &saveptr);
 	     component; component = next_component) {
@@ -191,10 +356,56 @@ glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
 		parent = inode;
 
 		inode = glfs_resolve_component (fs, subvol, parent,
-						component, &ciatt);
+						component, &ciatt,
+						/* force hard lookup on the last
+						   component, as the caller
+						   wants proper iatt filled
+						*/
+						(reval || (!next_component &&
+						iatt)));
 		if (!inode)
 			break;
 
+		if (IA_ISLNK (ciatt.ia_type) && (next_component || follow)) {
+			/* If the component is not the last piece,
+			   then following it is necessary even if
+			   not requested by the caller
+			*/
+			char *lpath = NULL;
+			loc_t sym_loc = {0,};
+
+			if (follow > GLFS_SYMLINK_MAX_FOLLOW) {
+				errno = ELOOP;
+				ret = -1;
+				if (inode) {
+					inode_unref (inode);
+					inode = NULL;
+				}
+				break;
+			}
+
+			ret = glfs_resolve_symlink (fs, subvol, inode, &lpath);
+			inode_unref (inode);
+			inode = NULL;
+			if (ret < 0)
+				break;
+
+			ret = glfs_resolve_at (fs, subvol, parent, lpath,
+					       &sym_loc,
+					       /* followed iatt becomes the
+						  component iatt
+					       */
+					       &ciatt,
+					       /* always recurisvely follow while
+						  following symlink
+					       */
+					       follow + 1, reval);
+			if (ret == 0)
+				inode = inode_ref (sym_loc.inode);
+			loc_wipe (&sym_loc);
+			GF_FREE (lpath);
+		}
+
 		if (!next_component)
 			break;
 
@@ -242,3 +453,517 @@ out:
 	return ret;
 }
 
+
+int
+glfs_resolve_path (struct glfs *fs, xlator_t *subvol, const char *origpath,
+		   loc_t *loc, struct iatt *iatt, int follow, int reval)
+{
+	int ret = -1;
+	inode_t *cwd = NULL;
+
+	if (origpath[0] == '/')
+		return glfs_resolve_at (fs, subvol, NULL, origpath, loc, iatt,
+					follow, reval);
+
+	cwd = glfs_cwd_get (fs);
+
+	ret = glfs_resolve_at (fs, subvol, cwd, origpath, loc, iatt,
+			       follow, reval);
+	if (cwd)
+		inode_unref (cwd);
+
+	return ret;
+}
+
+
+int
+glfs_resolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	      loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 1, reval);
+
+	return ret;
+}
+
+
+int
+glfs_lresolve (struct glfs *fs, xlator_t *subvol, const char *origpath,
+	       loc_t *loc, struct iatt *iatt, int reval)
+{
+	int ret = -1;
+
+	ret = glfs_resolve_path (fs, subvol, origpath, loc, iatt, 0, reval);
+
+	return ret;
+}
+
+
+int
+glfs_migrate_fd_locks_safe (struct glfs *fs, xlator_t *oldsubvol, fd_t *oldfd,
+			    xlator_t *newsubvol, fd_t *newfd)
+{
+	dict_t *lockinfo = NULL;
+	int ret = 0;
+	char uuid1[64];
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+	newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+	ret = syncop_fgetxattr (oldsubvol, oldfd, &lockinfo,
+				GF_XATTR_LOCKINFO_KEY);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fgetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	if (!dict_get (lockinfo, GF_XATTR_LOCKINFO_KEY)) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"missing lokinfo key (%s) on graph %s (%d)",
+			uuid_utoa_r (oldfd->inode->gfid, uuid1),
+			graphid_str (oldsubvol), oldsubvol->graph->id);
+		goto out;
+	}
+
+	ret = syncop_fsetxattr (newsubvol, newfd, lockinfo, 0);
+	if (ret < 0) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fsetxattr (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newfd->inode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+out:
+	if (lockinfo)
+		dict_unref (lockinfo);
+	return ret;
+}
+
+
+fd_t *
+glfs_migrate_fd_safe (struct glfs *fs, xlator_t *newsubvol, fd_t *oldfd)
+{
+	fd_t *newfd = NULL;
+	inode_t *oldinode = NULL;
+	inode_t *newinode = NULL;
+	xlator_t *oldsubvol = NULL;
+	int ret = -1;
+	loc_t loc = {0, };
+	char uuid1[64];
+
+
+	oldinode = oldfd->inode;
+	oldsubvol = oldinode->table->xl;
+
+	if (oldsubvol == newsubvol)
+		return fd_ref (oldfd);
+
+	if (!oldsubvol->switched) {
+		ret = syncop_fsync (oldsubvol, oldfd, 0);
+		if (ret) {
+			gf_log (fs->volname, GF_LOG_WARNING,
+				"fsync() failed (%s) on %s graph %s (%d)",
+				strerror (errno),
+				uuid_utoa_r (oldfd->inode->gfid, uuid1),
+				graphid_str (oldsubvol), oldsubvol->graph->id);
+		}
+	}
+
+	newinode = glfs_refresh_inode_safe (newsubvol, oldinode);
+	if (!newinode) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"inode (%s) refresh failed (%s) on graph %s (%d)",
+			uuid_utoa_r (oldinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	newfd = fd_create (newinode, getpid());
+	if (!newfd) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"fd_create (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	loc.inode = inode_ref (newinode);
+
+        ret = inode_path (oldfd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log (fs->volname, GF_LOG_INFO, "inode_path failed");
+                goto out;
+        }
+
+        uuid_copy (loc.gfid, oldinode->gfid);
+
+
+	if (IA_ISDIR (oldinode->ia_type))
+		ret = syncop_opendir (newsubvol, &loc, newfd);
+	else
+		ret = syncop_open (newsubvol, &loc,
+				   oldfd->flags & ~(O_TRUNC|O_EXCL|O_CREAT),
+				   newfd);
+	loc_wipe (&loc);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"syncop_open%s (%s) failed (%s) on graph %s (%d)",
+			IA_ISDIR (oldinode->ia_type) ? "dir" : "",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+	ret = glfs_migrate_fd_locks_safe (fs, oldsubvol, oldfd, newsubvol,
+					  newfd);
+
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_WARNING,
+			"lock migration (%s) failed (%s) on graph %s (%d)",
+			uuid_utoa_r (newinode->gfid, uuid1),
+			strerror (errno),
+			graphid_str (newsubvol), newsubvol->graph->id);
+		goto out;
+	}
+
+        newfd->flags = oldfd->flags;
+	fd_bind (newfd);
+out:
+	if (newinode)
+		inode_unref (newinode);
+
+	if (ret) {
+		fd_unref (newfd);
+		newfd = NULL;
+	}
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_migrate_fd (struct glfs *fs, xlator_t *newsubvol, struct glfs_fd *glfd)
+{
+	fd_t *oldfd = NULL;
+	fd_t *newfd = NULL;
+
+	oldfd = glfd->fd;
+
+	fs->migration_in_progress = 1;
+	pthread_mutex_unlock (&fs->mutex);
+	{
+		newfd = glfs_migrate_fd_safe (fs, newsubvol, oldfd);
+	}
+	pthread_mutex_lock (&fs->mutex);
+	fs->migration_in_progress = 0;
+	pthread_cond_broadcast (&fs->cond);
+
+	return newfd;
+}
+
+
+fd_t *
+__glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	if (glfd->fd->inode->table->xl == subvol)
+		return fd_ref (glfd->fd);
+
+	fd = __glfs_migrate_fd (fs, subvol, glfd);
+	if (!fd)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		fd_unref (glfd->fd);
+		glfd->fd = fd_ref (fd);
+	}
+
+	return fd;
+}
+
+
+fd_t *
+glfs_resolve_fd (struct glfs *fs, xlator_t *subvol, struct glfs_fd *glfd)
+{
+	fd_t *fd = NULL;
+
+	glfs_lock (fs);
+	{
+		fd = __glfs_resolve_fd (fs, subvol, glfd);
+	}
+	glfs_unlock (fs);
+
+	return fd;
+}
+
+
+void
+__glfs_migrate_openfds (struct glfs *fs, xlator_t *subvol)
+{
+	struct glfs_fd *glfd = NULL;
+	fd_t *fd = NULL;
+
+	list_for_each_entry (glfd, &fs->openfds, openfds) {
+		if (uuid_is_null (glfd->fd->inode->gfid)) {
+			gf_log (fs->volname, GF_LOG_INFO,
+				"skipping openfd %p/%p in graph %s (%d)",
+				glfd, glfd->fd,	graphid_str(subvol),
+				subvol->graph->id);
+			/* create in progress, defer */
+			continue;
+		}
+
+		fd = __glfs_migrate_fd (fs, subvol, glfd);
+		if (fd) {
+			fd_unref (glfd->fd);
+			glfd->fd = fd;
+		}
+	}
+}
+
+
+xlator_t *
+__glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *new_subvol = NULL;
+	int            ret = -1;
+	inode_t       *new_cwd = NULL;
+
+	if (!fs->next_subvol)
+		return fs->active_subvol;
+
+	new_subvol = fs->next_subvol;
+
+	ret = __glfs_first_lookup (fs, new_subvol);
+	if (ret) {
+		gf_log (fs->volname, GF_LOG_INFO,
+			"first lookup on graph %s (%d) failed (%s)",
+			graphid_str (new_subvol), new_subvol->graph->id,
+			strerror (errno));
+		return NULL;
+	}
+
+	if (fs->cwd) {
+		new_cwd = __glfs_refresh_inode (fs, new_subvol, fs->cwd);
+
+		if (!new_cwd) {
+			char buf1[64];
+			gf_log (fs->volname, GF_LOG_INFO,
+				"cwd refresh of %s graph %s (%d) failed (%s)",
+				uuid_utoa_r (fs->cwd->gfid, buf1),
+				graphid_str (new_subvol),
+				new_subvol->graph->id, strerror (errno));
+			return NULL;
+		}
+	}
+
+	__glfs_migrate_openfds (fs, new_subvol);
+
+	/* switching @active_subvol and @cwd
+	   should be atomic
+	*/
+	fs->old_subvol = fs->active_subvol;
+	fs->active_subvol = fs->next_subvol;
+	fs->next_subvol = NULL;
+
+	if (new_cwd) {
+		__glfs_cwd_set (fs, new_cwd);
+		inode_unref (new_cwd);
+	}
+
+	gf_log (fs->volname, GF_LOG_INFO, "switched to graph %s (%d)",
+		graphid_str (new_subvol), new_subvol->graph->id);
+
+	return new_subvol;
+}
+
+xlator_t *
+glfs_active_subvol (struct glfs *fs)
+{
+	xlator_t      *subvol = NULL;
+	xlator_t      *old_subvol = NULL;
+
+	glfs_lock (fs);
+	{
+		subvol = __glfs_active_subvol (fs);
+
+		if (subvol)
+			subvol->winds++;
+
+		if (fs->old_subvol) {
+			old_subvol = fs->old_subvol;
+			fs->old_subvol = NULL;
+			old_subvol->switched = 1;
+		}
+	}
+	glfs_unlock (fs);
+
+	if (old_subvol)
+		glfs_subvol_done (fs, old_subvol);
+
+	return subvol;
+}
+
+
+void
+glfs_subvol_done (struct glfs *fs, xlator_t *subvol)
+{
+	int ref = 0;
+	xlator_t *active_subvol = NULL;
+
+	if (!subvol)
+		return;
+
+	glfs_lock (fs);
+	{
+		ref = (--subvol->winds);
+		active_subvol = fs->active_subvol;
+	}
+	glfs_unlock (fs);
+
+	if (ref == 0) {
+		assert (subvol != active_subvol);
+		xlator_notify (subvol, GF_EVENT_PARENT_DOWN, subvol, NULL);
+	}
+}
+
+
+int
+__glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	if (inode->table->xl != fs->active_subvol) {
+		inode = __glfs_refresh_inode (fs, fs->active_subvol, inode);
+		if (!inode)
+			return -1;
+	} else {
+		inode_ref (inode);
+	}
+
+	if (fs->cwd)
+		inode_unref (fs->cwd);
+
+	fs->cwd = inode;
+
+	return 0;
+}
+
+
+int
+glfs_cwd_set (struct glfs *fs, inode_t *inode)
+{
+	int ret = 0;
+
+	glfs_lock (fs);
+	{
+		ret = __glfs_cwd_set (fs, inode);
+	}
+	glfs_unlock (fs);
+
+	return ret;
+}
+
+
+inode_t *
+__glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	if (!fs->cwd)
+		return NULL;
+
+	if (fs->cwd->table->xl == fs->active_subvol) {
+		cwd = inode_ref (fs->cwd);
+		return cwd;
+	}
+
+	cwd = __glfs_refresh_inode (fs, fs->active_subvol, fs->cwd);
+
+	return cwd;
+}
+
+inode_t *
+glfs_cwd_get (struct glfs *fs)
+{
+	inode_t *cwd = NULL;
+
+	glfs_lock (fs);
+	{
+		cwd = __glfs_cwd_get (fs);
+	}
+	glfs_unlock (fs);
+
+	return cwd;
+}
+
+inode_t *
+__glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	if (object->inode->table->xl == subvol)
+		return inode_ref (object->inode);
+
+	inode = __glfs_refresh_inode (fs, fs->active_subvol,
+					object->inode);
+	if (!inode)
+		return NULL;
+
+	if (subvol == fs->active_subvol) {
+		inode_unref (object->inode);
+		object->inode = inode_ref (inode);
+	}
+
+	return inode;
+}
+
+inode_t *
+glfs_resolve_inode (struct glfs *fs, xlator_t *subvol,
+		    struct glfs_object *object)
+{
+	inode_t *inode = NULL;
+
+	glfs_lock (fs);
+	{
+		inode = __glfs_resolve_inode(fs, subvol, object);
+	}
+	glfs_unlock (fs);
+
+	return inode;
+}
+
+int
+glfs_create_object (loc_t *loc, struct glfs_object **retobject)
+{
+	struct glfs_object *object = NULL;
+
+	object = GF_CALLOC (1, sizeof(struct glfs_object),
+			    glfs_mt_glfs_object_t);
+	if (object == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	object->inode = loc->inode;
+	uuid_copy (object->gfid, object->inode->gfid);
+
+	/* we hold the reference */
+	loc->inode = NULL;
+
+	*retobject = object;
+
+	return 0;
+}
diff --git a/api/src/glfs.c b/api/src/glfs.c
index 6b3c2113d..29ed47c0c 100644
--- a/api/src/glfs.c
+++ b/api/src/glfs.c
@@ -11,19 +11,15 @@
 
 /*
   TODO:
+  - merge locks in glfs_posix_lock for lock self-healing
   - set proper pid/lk_owner to call frames (currently buried in syncop)
   - fix logging.c/h to store logfp and loglevel in glusterfs_ctx_t and
     reach it via THIS.
-  - fd migration on graph switch.
   - update syncop functions to accept/return xdata. ???
-  - syncop_readv to not touch params if args.op_ret < 0.
   - protocol/client to reconnect immediately after portmap disconnect.
   - handle SEEK_END failure in _lseek()
   - handle umask (per filesystem?)
-  - implement glfs_set_xlator_option(), like --xlator-option
   - make itables LRU based
-  - implement glfs_fini()
-  - modify syncop_fsync() to accept 'dataonly' flag
   - 0-copy for readv/writev
   - reconcile the open/creat mess
 */
@@ -53,6 +49,8 @@
 
 #include "glfs.h"
 #include "glfs-internal.h"
+#include "hashfn.h"
+#include "rpc-clnt.h"
 
 
 static gf_boolean_t
@@ -68,7 +66,7 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 	call_pool_t   *pool = NULL;
 	int	       ret = -1;
 
-	xlator_mem_acct_init (THIS, glfs_mt_end);
+	xlator_mem_acct_init (THIS, glfs_mt_end + 1);
 
 	ctx->process_uuid = generate_glusterfs_ctx_id ();
 	if (!ctx->process_uuid) {
@@ -87,7 +85,7 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
 		goto err;
 	}
 
-	ctx->env = syncenv_new (0);
+	ctx->env = syncenv_new (0, 0, 0);
 	if (!ctx->env) {
 		goto err;
 	}
@@ -279,69 +277,117 @@ out:
 ///////////////////////////////////////////////////////////////////////////////
 
 
-struct glfs *
-glfs_from_glfd (struct glfs_fd *glfd)
+int
+glfs_set_xlator_option (struct glfs *fs, const char *xlator, const char *key,
+			const char *value)
 {
-	return 	((xlator_t *)glfd->fd->inode->table->xl->ctx->master)->private;
+	xlator_cmdline_option_t *option = NULL;
+
+	option = GF_CALLOC (1, sizeof (*option),
+			    glfs_mt_xlator_cmdline_option_t);
+	if (!option)
+		goto enomem;
+
+	INIT_LIST_HEAD (&option->cmd_args);
+
+	option->volume = gf_strdup (xlator);
+	if (!option->volume)
+		goto enomem;
+	option->key = gf_strdup (key);
+	if (!option->key)
+		goto enomem;
+	option->value = gf_strdup (value);
+	if (!option->value)
+		goto enomem;
+
+	list_add (&option->cmd_args, &fs->ctx->cmd_args.xlator_options);
+
+	return 0;
+enomem:
+	errno = ENOMEM;
+
+	if (!option)
+		return -1;
+
+	GF_FREE (option->volume);
+	GF_FREE (option->key);
+	GF_FREE (option->value);
+	GF_FREE (option);
+
+	return -1;
 }
 
+int glfs_setfsuid (uid_t fsuid)
+{
+	return syncopctx_setfsuid (&fsuid);
+}
 
-void
-glfs_fd_destroy (struct glfs_fd *glfd)
+int glfs_setfsgid (gid_t fsgid)
 {
-	if (!glfd)
-		return;
-	if (glfd->fd)
-		fd_unref (glfd->fd);
-	GF_FREE (glfd);
+	return syncopctx_setfsgid (&fsgid);
 }
 
+int glfs_setfsgroups (size_t size, const gid_t *list)
+{
+	return syncopctx_setfsgroups(size, list);
+}
 
-xlator_t *
-glfs_fd_subvol (struct glfs_fd *glfd)
+struct glfs *
+glfs_from_glfd (struct glfs_fd *glfd)
 {
-	xlator_t    *subvol = NULL;
+	return glfd->fs;
+}
+
+
+struct glfs_fd *
+glfs_fd_new (struct glfs *fs)
+{
+	struct glfs_fd  *glfd = NULL;
 
+	glfd = GF_CALLOC (1, sizeof (*glfd), glfs_mt_glfs_fd_t);
 	if (!glfd)
 		return NULL;
 
-	subvol = glfd->fd->inode->table->xl;
+	glfd->fs = fs;
 
-	return subvol;
+	INIT_LIST_HEAD (&glfd->openfds);
+
+	return glfd;
 }
 
 
-xlator_t *
-glfs_active_subvol (struct glfs *fs)
+void
+glfs_fd_bind (struct glfs_fd *glfd)
 {
-	xlator_t      *subvol = NULL;
-	inode_table_t *itable = NULL;
+	struct glfs *fs = NULL;
 
-	pthread_mutex_lock (&fs->mutex);
-	{
-		while (!fs->init)
-			pthread_cond_wait (&fs->cond, &fs->mutex);
+	fs = glfd->fs;
 
-		subvol = fs->active_subvol;
+	glfs_lock (fs);
+	{
+		list_add_tail (&glfd->openfds, &fs->openfds);
 	}
-	pthread_mutex_unlock (&fs->mutex);
+	glfs_unlock (fs);
+}
 
-	if (!subvol)
-		return NULL;
+void
+glfs_fd_destroy (struct glfs_fd *glfd)
+{
+	if (!glfd)
+		return;
 
-	if (!subvol->itable) {
-		itable = inode_table_new (0, subvol);
-		if (!itable) {
-			errno = ENOMEM;
-			return NULL;
-		}
+	glfs_lock (glfd->fs);
+	{
+		list_del_init (&glfd->openfds);
+	}
+	glfs_unlock (glfd->fs);
 
-		subvol->itable = itable;
+	if (glfd->fd)
+		fd_unref (glfd->fd);
 
-		glfs_first_lookup (subvol);
-	}
+	GF_FREE (glfd->readdirbuf);
 
-	return subvol;
+	GF_FREE (glfd);
 }
 
 
@@ -370,7 +416,9 @@ glfs_new (const char *volname)
 		return NULL;
 	}
 
+#ifdef DEBUG
         gf_mem_acct_enable_set (ctx);
+#endif
 
 	/* first globals init, for gf_mem_acct_enable_set () */
 	ret = glusterfs_globals_init (ctx);
@@ -398,6 +446,8 @@ glfs_new (const char *volname)
 	pthread_mutex_init (&fs->mutex, NULL);
 	pthread_cond_init (&fs->cond, NULL);
 
+	INIT_LIST_HEAD (&fs->openfds);
+
 	return fs;
 }
 
@@ -441,14 +491,26 @@ glfs_set_volfile_server (struct glfs *fs, const char *transport,
 int
 glfs_set_logging (struct glfs *fs, const char *logfile, int loglevel)
 {
-	int  ret = -1;
+	int  ret = 0;
+        char *tmplog = NULL;
 
-	ret = gf_log_init (fs->ctx, logfile);
-	if (ret)
-		return ret;
+        if (!logfile) {
+                ret = gf_set_log_file_path (&fs->ctx->cmd_args);
+                if (ret)
+                        goto out;
+                tmplog = fs->ctx->cmd_args.log_file;
+        } else {
+                tmplog = (char *)logfile;
+        }
 
-	gf_log_set_loglevel (loglevel);
+        ret = gf_log_init (fs->ctx, tmplog, NULL);
+        if (ret)
+                goto out;
 
+	if (loglevel >= 0)
+		gf_log_set_loglevel (loglevel);
+
+out:
 	return ret;
 }
 
@@ -458,7 +520,8 @@ glfs_init_wait (struct glfs *fs)
 {
 	int   ret = -1;
 
-	pthread_mutex_lock (&fs->mutex);
+	/* Always a top-down call, use glfs_lock() */
+	glfs_lock (fs);
 	{
 		while (!fs->init)
 			pthread_cond_wait (&fs->cond,
@@ -466,7 +529,7 @@ glfs_init_wait (struct glfs *fs)
 		ret = fs->ret;
 		errno = fs->err;
 	}
-	pthread_mutex_unlock (&fs->mutex);
+	glfs_unlock (fs);
 
 	return ret;
 }
@@ -475,20 +538,32 @@ glfs_init_wait (struct glfs *fs)
 void
 glfs_init_done (struct glfs *fs, int ret)
 {
-	if (fs->init_cbk) {
-		fs->init_cbk (fs, ret);
-		return;
+	glfs_init_cbk init_cbk;
+
+	if (!fs) {
+		gf_log ("glfs", GF_LOG_ERROR,
+			"fs is NULL");
+		goto out;
 	}